2024-11-07 12:22:55,841 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:22:55,855 main DEBUG Took 0.012148 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 12:22:55,855 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 12:22:55,856 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 12:22:55,857 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 12:22:55,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,866 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 12:22:55,878 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,880 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,880 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,881 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,881 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,882 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,882 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,883 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,883 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,884 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,884 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,884 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,885 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,885 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,886 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,886 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,886 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,887 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,887 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,887 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,888 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,888 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,888 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 12:22:55,889 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,889 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 12:22:55,891 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 12:22:55,892 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 12:22:55,895 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 12:22:55,896 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 12:22:55,897 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 12:22:55,898 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 12:22:55,908 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 12:22:55,911 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 12:22:55,913 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 12:22:55,914 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 12:22:55,914 main DEBUG createAppenders(={Console}) 2024-11-07 12:22:55,915 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 12:22:55,915 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 12:22:55,915 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 12:22:55,916 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 12:22:55,916 main DEBUG OutputStream closed 2024-11-07 12:22:55,917 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 12:22:55,917 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 12:22:55,917 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 12:22:56,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 12:22:56,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 12:22:56,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 12:22:56,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 12:22:56,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 12:22:56,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 12:22:56,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 12:22:56,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 12:22:56,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 12:22:56,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 12:22:56,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 12:22:56,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 12:22:56,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 12:22:56,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 12:22:56,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 12:22:56,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 12:22:56,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 12:22:56,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 12:22:56,044 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 12:22:56,045 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 12:22:56,045 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 12:22:56,047 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T12:22:56,417 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5 2024-11-07 12:22:56,421 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 12:22:56,422 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T12:22:56,432 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T12:22:56,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T12:22:56,576 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859, deleteOnExit=true 2024-11-07T12:22:56,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T12:22:56,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/test.cache.data in system properties and HBase conf 2024-11-07T12:22:56,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T12:22:56,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.log.dir in system properties and HBase conf 2024-11-07T12:22:56,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T12:22:56,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T12:22:56,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T12:22:56,690 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T12:22:56,769 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T12:22:56,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:22:56,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T12:22:56,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T12:22:56,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:22:56,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T12:22:56,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T12:22:56,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T12:22:56,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:22:56,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T12:22:56,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/nfs.dump.dir in system properties and HBase conf 2024-11-07T12:22:56,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/java.io.tmpdir in system properties and HBase conf 2024-11-07T12:22:56,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T12:22:56,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T12:22:56,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T12:22:57,927 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T12:22:58,009 INFO [Time-limited test {}] log.Log(170): Logging initialized @3152ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T12:22:58,086 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:22:58,151 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:22:58,170 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:22:58,170 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:22:58,171 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T12:22:58,181 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:22:58,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:22:58,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:22:58,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/java.io.tmpdir/jetty-localhost-34559-hadoop-hdfs-3_4_1-tests_jar-_-any-6302395337413867734/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:22:58,386 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:34559} 2024-11-07T12:22:58,386 INFO [Time-limited test {}] server.Server(415): Started @3530ms 2024-11-07T12:22:58,991 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T12:22:58,998 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T12:22:58,999 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T12:22:58,999 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T12:22:59,000 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T12:22:59,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.log.dir/,AVAILABLE} 2024-11-07T12:22:59,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T12:22:59,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/java.io.tmpdir/jetty-localhost-45595-hadoop-hdfs-3_4_1-tests_jar-_-any-16780647261893588274/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:22:59,106 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:45595} 2024-11-07T12:22:59,106 INFO [Time-limited test {}] server.Server(415): Started @4250ms 2024-11-07T12:22:59,174 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T12:23:00,380 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data1/current/BP-1409914116-172.17.0.2-1730982177397/current, will proceed with Du for space computation calculation, 2024-11-07T12:23:00,380 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data2/current/BP-1409914116-172.17.0.2-1730982177397/current, will proceed with Du for space computation calculation, 2024-11-07T12:23:00,455 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T12:23:00,493 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5 2024-11-07T12:23:00,499 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46b79b6754212a with lease ID 0xb9c027c401975f25: Processing first storage report for DS-87e581ec-bca6-46e6-8209-e2a77d78838f from datanode DatanodeRegistration(127.0.0.1:43943, datanodeUuid=e5e0a903-46f3-4733-8cee-0a5f90087d4c, infoPort=35907, infoSecurePort=0, ipcPort=39069, storageInfo=lv=-57;cid=testClusterID;nsid=1765973684;c=1730982177397) 2024-11-07T12:23:00,500 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46b79b6754212a with lease ID 0xb9c027c401975f25: from storage DS-87e581ec-bca6-46e6-8209-e2a77d78838f node DatanodeRegistration(127.0.0.1:43943, datanodeUuid=e5e0a903-46f3-4733-8cee-0a5f90087d4c, infoPort=35907, infoSecurePort=0, ipcPort=39069, storageInfo=lv=-57;cid=testClusterID;nsid=1765973684;c=1730982177397), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T12:23:00,501 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46b79b6754212a with lease ID 0xb9c027c401975f25: Processing first storage report for DS-3567d93a-bdd3-42e3-a110-5311523cbe6b from datanode DatanodeRegistration(127.0.0.1:43943, datanodeUuid=e5e0a903-46f3-4733-8cee-0a5f90087d4c, infoPort=35907, infoSecurePort=0, ipcPort=39069, storageInfo=lv=-57;cid=testClusterID;nsid=1765973684;c=1730982177397) 2024-11-07T12:23:00,501 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46b79b6754212a with lease ID 0xb9c027c401975f25: from storage DS-3567d93a-bdd3-42e3-a110-5311523cbe6b node DatanodeRegistration(127.0.0.1:43943, datanodeUuid=e5e0a903-46f3-4733-8cee-0a5f90087d4c, infoPort=35907, infoSecurePort=0, ipcPort=39069, storageInfo=lv=-57;cid=testClusterID;nsid=1765973684;c=1730982177397), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T12:23:00,560 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/zookeeper_0, clientPort=51223, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T12:23:00,571 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51223 2024-11-07T12:23:00,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:00,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741825_1001 (size=7) 2024-11-07T12:23:01,226 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 with version=8 2024-11-07T12:23:01,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/hbase-staging 2024-11-07T12:23:01,329 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T12:23:01,586 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8944cbb06cd2:0 server-side Connection retries=45 2024-11-07T12:23:01,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:01,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:01,601 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:23:01,601 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:01,601 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:23:01,761 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T12:23:01,816 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T12:23:01,826 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T12:23:01,829 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:23:01,854 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18486 (auto-detected) 2024-11-07T12:23:01,855 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T12:23:01,872 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37285 2024-11-07T12:23:01,897 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37285 connecting to ZooKeeper ensemble=127.0.0.1:51223 2024-11-07T12:23:02,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:372850x0, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:23:02,018 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37285-0x101153ff26c0000 connected 2024-11-07T12:23:02,105 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:02,108 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:02,118 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:23:02,121 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8, hbase.cluster.distributed=false 2024-11-07T12:23:02,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:23:02,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37285 2024-11-07T12:23:02,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37285 2024-11-07T12:23:02,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37285 2024-11-07T12:23:02,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37285 2024-11-07T12:23:02,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37285 2024-11-07T12:23:02,254 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8944cbb06cd2:0 server-side Connection retries=45 2024-11-07T12:23:02,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:02,255 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:02,256 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T12:23:02,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T12:23:02,256 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T12:23:02,258 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T12:23:02,261 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T12:23:02,262 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46221 2024-11-07T12:23:02,264 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46221 connecting to ZooKeeper ensemble=127.0.0.1:51223 2024-11-07T12:23:02,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:02,268 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:02,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462210x0, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T12:23:02,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:23:02,282 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46221-0x101153ff26c0001 connected 2024-11-07T12:23:02,286 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T12:23:02,294 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T12:23:02,297 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T12:23:02,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T12:23:02,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46221 2024-11-07T12:23:02,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46221 2024-11-07T12:23:02,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46221 2024-11-07T12:23:02,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46221 2024-11-07T12:23:02,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46221 2024-11-07T12:23:02,325 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8944cbb06cd2:37285 2024-11-07T12:23:02,326 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:02,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:23:02,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:23:02,347 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T12:23:02,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:02,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:02,378 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T12:23:02,379 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8944cbb06cd2,37285,1730982181418 from backup master directory 2024-11-07T12:23:02,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:02,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:23:02,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T12:23:02,388 WARN [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:23:02,388 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:02,390 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T12:23:02,391 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T12:23:02,444 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/hbase.id] with ID: 59d2ef22-9ddc-4e31-b944-862110ef10b2 2024-11-07T12:23:02,444 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/.tmp/hbase.id 2024-11-07T12:23:02,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741826_1002 (size=42) 2024-11-07T12:23:02,858 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/.tmp/hbase.id]:[hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/hbase.id] 2024-11-07T12:23:02,916 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T12:23:02,921 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T12:23:02,941 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T12:23:02,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:02,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:03,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741827_1003 (size=196) 2024-11-07T12:23:03,431 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:23:03,433 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T12:23:03,447 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:03,451 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:23:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741828_1004 (size=1189) 2024-11-07T12:23:03,897 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store 2024-11-07T12:23:03,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741829_1005 (size=34) 2024-11-07T12:23:04,320 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T12:23:04,323 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:04,325 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:23:04,325 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:23:04,326 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:23:04,328 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:23:04,328 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:23:04,328 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:23:04,330 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730982184325Disabling compacts and flushes for region at 1730982184325Disabling writes for close at 1730982184328 (+3 ms)Writing region close event to WAL at 1730982184328Closed at 1730982184328 2024-11-07T12:23:04,332 WARN [master/8944cbb06cd2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/.initializing 2024-11-07T12:23:04,333 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/WALs/8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:04,342 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:23:04,358 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8944cbb06cd2%2C37285%2C1730982181418, suffix=, logDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/WALs/8944cbb06cd2,37285,1730982181418, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/oldWALs, maxLogs=10 2024-11-07T12:23:04,379 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/WALs/8944cbb06cd2,37285,1730982181418/8944cbb06cd2%2C37285%2C1730982181418.1730982184362, exclude list is [], retry=0 2024-11-07T12:23:04,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43943,DS-87e581ec-bca6-46e6-8209-e2a77d78838f,DISK] 2024-11-07T12:23:04,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T12:23:04,428 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/WALs/8944cbb06cd2,37285,1730982181418/8944cbb06cd2%2C37285%2C1730982181418.1730982184362 2024-11-07T12:23:04,429 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35907:35907)] 2024-11-07T12:23:04,430 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:23:04,430 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:04,434 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,435 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T12:23:04,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:04,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:04,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T12:23:04,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:04,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:04,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,506 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T12:23:04,506 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:04,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:04,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T12:23:04,510 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:04,511 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:04,512 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,515 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,516 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,521 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,522 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,526 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T12:23:04,531 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T12:23:04,536 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:23:04,538 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60129453, jitterRate=-0.10400132834911346}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T12:23:04,544 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730982184448Initializing all the Stores at 1730982184450 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982184451 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982184451Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982184452 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982184452Cleaning up temporary data from old regions at 1730982184522 (+70 ms)Region opened successfully at 1730982184544 (+22 ms) 2024-11-07T12:23:04,545 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T12:23:04,575 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79110a47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8944cbb06cd2/172.17.0.2:0 2024-11-07T12:23:04,602 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T12:23:04,611 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T12:23:04,612 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T12:23:04,614 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T12:23:04,615 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T12:23:04,620 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T12:23:04,620 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T12:23:04,649 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T12:23:04,660 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T12:23:04,724 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T12:23:04,727 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T12:23:04,729 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T12:23:04,745 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T12:23:04,747 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T12:23:04,751 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T12:23:04,755 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T12:23:04,757 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T12:23:04,765 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T12:23:04,783 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T12:23:04,786 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T12:23:04,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:23:04,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T12:23:04,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,801 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8944cbb06cd2,37285,1730982181418, sessionid=0x101153ff26c0000, setting cluster-up flag (Was=false) 2024-11-07T12:23:04,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,860 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T12:23:04,863 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:04,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:04,913 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T12:23:04,915 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:04,921 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T12:23:04,989 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T12:23:05,000 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T12:23:05,006 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T12:23:05,011 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8944cbb06cd2,37285,1730982181418 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T12:23:05,016 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(746): ClusterId : 59d2ef22-9ddc-4e31-b944-862110ef10b2 2024-11-07T12:23:05,018 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8944cbb06cd2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:23:05,018 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8944cbb06cd2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:23:05,018 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8944cbb06cd2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:23:05,018 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8944cbb06cd2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T12:23:05,018 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8944cbb06cd2:0, corePoolSize=10, maxPoolSize=10 2024-11-07T12:23:05,018 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T12:23:05,019 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,019 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8944cbb06cd2:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:23:05,019 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,021 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730982215020 2024-11-07T12:23:05,022 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T12:23:05,024 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:23:05,024 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T12:23:05,024 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T12:23:05,028 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T12:23:05,029 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T12:23:05,029 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T12:23:05,029 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T12:23:05,030 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:05,030 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T12:23:05,031 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T12:23:05,031 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T12:23:05,032 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,036 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T12:23:05,037 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T12:23:05,038 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T12:23:05,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741831_1007 (size=1321) 2024-11-07T12:23:05,041 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T12:23:05,041 DEBUG [RS:0;8944cbb06cd2:46221 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24edb8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8944cbb06cd2/172.17.0.2:0 2024-11-07T12:23:05,044 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T12:23:05,045 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T12:23:05,046 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.large.0-1730982185046,5,FailOnTimeoutGroup] 2024-11-07T12:23:05,047 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.small.0-1730982185047,5,FailOnTimeoutGroup] 2024-11-07T12:23:05,047 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,047 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T12:23:05,048 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,049 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,056 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8944cbb06cd2:46221 2024-11-07T12:23:05,059 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T12:23:05,059 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T12:23:05,059 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T12:23:05,062 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(2659): reportForDuty to master=8944cbb06cd2,37285,1730982181418 with port=46221, startcode=1730982182218 2024-11-07T12:23:05,072 DEBUG [RS:0;8944cbb06cd2:46221 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T12:23:05,104 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44075, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T12:23:05,110 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37285 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,112 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37285 {}] master.ServerManager(517): Registering regionserver=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,125 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:23:05,125 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41983 2024-11-07T12:23:05,125 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T12:23:05,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:23:05,135 DEBUG [RS:0;8944cbb06cd2:46221 {}] zookeeper.ZKUtil(111): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,135 WARN [RS:0;8944cbb06cd2:46221 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T12:23:05,135 INFO [RS:0;8944cbb06cd2:46221 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:23:05,135 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,137 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8944cbb06cd2,46221,1730982182218] 2024-11-07T12:23:05,162 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T12:23:05,182 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T12:23:05,186 INFO [RS:0;8944cbb06cd2:46221 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T12:23:05,187 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,187 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T12:23:05,193 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T12:23:05,194 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,194 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,195 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,195 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,195 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,195 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,195 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8944cbb06cd2:0, corePoolSize=2, maxPoolSize=2 2024-11-07T12:23:05,196 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,196 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,196 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,196 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,197 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,197 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8944cbb06cd2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T12:23:05,197 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8944cbb06cd2:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:23:05,197 DEBUG [RS:0;8944cbb06cd2:46221 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0, corePoolSize=3, maxPoolSize=3 2024-11-07T12:23:05,198 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,198 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,199 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,199 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,199 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,199 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,46221,1730982182218-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:23:05,216 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T12:23:05,218 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,46221,1730982182218-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,218 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,219 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.Replication(171): 8944cbb06cd2,46221,1730982182218 started 2024-11-07T12:23:05,237 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:05,237 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1482): Serving as 8944cbb06cd2,46221,1730982182218, RpcServer on 8944cbb06cd2/172.17.0.2:46221, sessionid=0x101153ff26c0001 2024-11-07T12:23:05,238 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T12:23:05,239 DEBUG [RS:0;8944cbb06cd2:46221 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,239 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8944cbb06cd2,46221,1730982182218' 2024-11-07T12:23:05,239 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T12:23:05,240 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T12:23:05,241 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T12:23:05,241 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T12:23:05,241 DEBUG [RS:0;8944cbb06cd2:46221 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:05,241 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8944cbb06cd2,46221,1730982182218' 2024-11-07T12:23:05,242 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T12:23:05,242 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T12:23:05,243 DEBUG [RS:0;8944cbb06cd2:46221 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T12:23:05,243 INFO [RS:0;8944cbb06cd2:46221 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T12:23:05,243 INFO [RS:0;8944cbb06cd2:46221 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T12:23:05,350 INFO [RS:0;8944cbb06cd2:46221 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T12:23:05,354 INFO [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8944cbb06cd2%2C46221%2C1730982182218, suffix=, logDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/oldWALs, maxLogs=32 2024-11-07T12:23:05,368 DEBUG [RS:0;8944cbb06cd2:46221 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218/8944cbb06cd2%2C46221%2C1730982182218.1730982185357, exclude list is [], retry=0 2024-11-07T12:23:05,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43943,DS-87e581ec-bca6-46e6-8209-e2a77d78838f,DISK] 2024-11-07T12:23:05,376 INFO [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218/8944cbb06cd2%2C46221%2C1730982182218.1730982185357 2024-11-07T12:23:05,377 DEBUG [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35907:35907)] 2024-11-07T12:23:05,445 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T12:23:05,446 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:23:05,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741833_1009 (size=32) 2024-11-07T12:23:05,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:05,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:23:05,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:23:05,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:05,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:05,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:23:05,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:23:05,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:05,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:05,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:23:05,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:23:05,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:05,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:05,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:23:05,882 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:23:05,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:05,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:05,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:23:05,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740 2024-11-07T12:23:05,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740 2024-11-07T12:23:05,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:23:05,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:23:05,889 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:23:05,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:23:05,894 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:23:05,895 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64682049, jitterRate=-0.0361623615026474}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:23:05,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730982185865Initializing all the Stores at 1730982185868 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982185868Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982185868Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982185868Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982185869 (+1 ms)Cleaning up temporary data from old regions at 1730982185888 (+19 ms)Region opened successfully at 1730982185898 (+10 ms) 2024-11-07T12:23:05,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:23:05,899 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:23:05,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:23:05,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:23:05,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:23:05,900 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:23:05,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730982185898Disabling compacts and flushes for region at 1730982185898Disabling writes for close at 1730982185899 (+1 ms)Writing region close event to WAL at 1730982185900 (+1 ms)Closed at 1730982185900 2024-11-07T12:23:05,903 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:23:05,903 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T12:23:05,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T12:23:05,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:23:05,918 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T12:23:06,071 DEBUG [8944cbb06cd2:37285 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T12:23:06,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:06,090 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8944cbb06cd2,46221,1730982182218, state=OPENING 2024-11-07T12:23:06,266 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T12:23:06,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:06,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:23:06,293 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:23:06,293 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:23:06,295 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T12:23:06,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:06,475 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T12:23:06,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51625, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T12:23:06,492 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T12:23:06,492 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T12:23:06,493 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T12:23:06,496 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8944cbb06cd2%2C46221%2C1730982182218.meta, suffix=.meta, logDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/oldWALs, maxLogs=32 2024-11-07T12:23:06,510 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218/8944cbb06cd2%2C46221%2C1730982182218.meta.1730982186497.meta, exclude list is [], retry=0 2024-11-07T12:23:06,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43943,DS-87e581ec-bca6-46e6-8209-e2a77d78838f,DISK] 2024-11-07T12:23:06,517 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/WALs/8944cbb06cd2,46221,1730982182218/8944cbb06cd2%2C46221%2C1730982182218.meta.1730982186497.meta 2024-11-07T12:23:06,518 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35907:35907)] 2024-11-07T12:23:06,518 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:23:06,520 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T12:23:06,522 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T12:23:06,526 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T12:23:06,530 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T12:23:06,531 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:06,531 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T12:23:06,531 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T12:23:06,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T12:23:06,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T12:23:06,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:06,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:06,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T12:23:06,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T12:23:06,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:06,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:06,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T12:23:06,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T12:23:06,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:06,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:06,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T12:23:06,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T12:23:06,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:06,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T12:23:06,546 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T12:23:06,548 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740 2024-11-07T12:23:06,551 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740 2024-11-07T12:23:06,554 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T12:23:06,554 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T12:23:06,555 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:23:06,559 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T12:23:06,561 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65334692, jitterRate=-0.026437222957611084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:23:06,561 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T12:23:06,563 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730982186532Writing region info on filesystem at 1730982186532Initializing all the Stores at 1730982186534 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982186534Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982186535 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982186535Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730982186535Cleaning up temporary data from old regions at 1730982186554 (+19 ms)Running coprocessor post-open hooks at 1730982186561 (+7 ms)Region opened successfully at 1730982186563 (+2 ms) 2024-11-07T12:23:06,570 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730982186467 2024-11-07T12:23:06,581 DEBUG [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T12:23:06,582 INFO [RS_OPEN_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T12:23:06,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:06,586 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8944cbb06cd2,46221,1730982182218, state=OPEN 2024-11-07T12:23:06,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:23:06,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T12:23:06,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:23:06,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T12:23:06,673 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:06,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T12:23:06,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8944cbb06cd2,46221,1730982182218 in 376 msec 2024-11-07T12:23:06,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T12:23:06,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 773 msec 2024-11-07T12:23:06,688 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T12:23:06,689 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T12:23:06,708 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:06,709 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:06,729 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:06,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:06,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8000 sec 2024-11-07T12:23:06,751 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730982186751, completionTime=-1 2024-11-07T12:23:06,754 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T12:23:06,754 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T12:23:06,778 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-07T12:23:06,778 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730982246778 2024-11-07T12:23:06,778 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730982306778 2024-11-07T12:23:06,778 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-07T12:23:06,780 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,781 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,781 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,782 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8944cbb06cd2:37285, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,782 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,783 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,788 DEBUG [master/8944cbb06cd2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T12:23:06,811 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.423sec 2024-11-07T12:23:06,812 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T12:23:06,813 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T12:23:06,814 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T12:23:06,815 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T12:23:06,815 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T12:23:06,816 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T12:23:06,816 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T12:23:06,825 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T12:23:06,826 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T12:23:06,827 INFO [master/8944cbb06cd2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8944cbb06cd2,37285,1730982181418-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T12:23:06,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:06,829 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T12:23:06,829 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T12:23:06,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:06,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:06,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:06,853 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:06,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:06,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:06,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:06,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:06,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=769, ProcessCount=11, AvailableMemoryMB=2250 2024-11-07T12:23:06,913 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:23:06,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-07T12:23:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:23:06,928 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:23:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:23:06,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:23:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:06,947 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:23:06,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T12:23:06,949 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:06,952 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:23:06,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:23:06,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741835_1011 (size=959) 2024-11-07T12:23:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:23:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:23:07,370 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:23:07,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741836_1012 (size=53) 2024-11-07T12:23:07,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:07,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 533db45226dd865e0c0b2154b214ffef, disabling compactions & flushes 2024-11-07T12:23:07,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. after waiting 0 ms 2024-11-07T12:23:07,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 533db45226dd865e0c0b2154b214ffef: Waiting for close lock at 1730982187380Disabling compacts and flushes for region at 1730982187380Disabling writes for close at 1730982187381 (+1 ms)Writing region close event to WAL at 1730982187381Closed at 1730982187381 2024-11-07T12:23:07,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:23:07,390 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982187384"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982187384"}]},"ts":"1730982187384"} 2024-11-07T12:23:07,395 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:23:07,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:23:07,400 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982187398"}]},"ts":"1730982187398"} 2024-11-07T12:23:07,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:23:07,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, ASSIGN}] 2024-11-07T12:23:07,409 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, ASSIGN 2024-11-07T12:23:07,411 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:23:07,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=533db45226dd865e0c0b2154b214ffef, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:07,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, ASSIGN because future has completed 2024-11-07T12:23:07,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:23:07,733 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,733 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:23:07,733 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,734 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:07,734 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,734 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,736 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,739 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 533db45226dd865e0c0b2154b214ffef columnFamilyName A 2024-11-07T12:23:07,739 DEBUG [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:07,741 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(327): Store=533db45226dd865e0c0b2154b214ffef/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:07,741 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,744 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 533db45226dd865e0c0b2154b214ffef columnFamilyName B 2024-11-07T12:23:07,744 DEBUG [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:07,745 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(327): Store=533db45226dd865e0c0b2154b214ffef/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:07,746 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,748 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 533db45226dd865e0c0b2154b214ffef columnFamilyName C 2024-11-07T12:23:07,749 DEBUG [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:07,750 INFO [StoreOpener-533db45226dd865e0c0b2154b214ffef-1 {}] regionserver.HStore(327): Store=533db45226dd865e0c0b2154b214ffef/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:07,750 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,753 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,754 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,756 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,757 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,758 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:23:07,761 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,766 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:23:07,767 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 533db45226dd865e0c0b2154b214ffef; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72734407, jitterRate=0.08382712304592133}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:23:07,767 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:07,769 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 533db45226dd865e0c0b2154b214ffef: Running coprocessor pre-open hook at 1730982187734Writing region info on filesystem at 1730982187734Initializing all the Stores at 1730982187736 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982187736Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982187736Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982187736Cleaning up temporary data from old regions at 1730982187757 (+21 ms)Running coprocessor post-open hooks at 1730982187767 (+10 ms)Region opened successfully at 1730982187769 (+2 ms) 2024-11-07T12:23:07,772 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., pid=6, masterSystemTime=1730982187726 2024-11-07T12:23:07,776 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,776 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:07,778 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=533db45226dd865e0c0b2154b214ffef, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:07,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:23:07,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T12:23:07,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 in 215 msec 2024-11-07T12:23:07,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T12:23:07,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, ASSIGN in 386 msec 2024-11-07T12:23:07,800 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:23:07,800 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982187800"}]},"ts":"1730982187800"} 2024-11-07T12:23:07,804 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:23:07,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:23:07,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 869 msec 2024-11-07T12:23:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T12:23:08,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:08,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,116 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ca7478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,120 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38922b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,122 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,130 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:23:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28ec11fd 2024-11-07T12:23:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:23:08,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:23:08,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c5ef3e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43659da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,153 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c72f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745237a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,162 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174557e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,171 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77447b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b988aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,178 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cda1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,184 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffa2de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d3b492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,191 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f9dc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1f4fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27451c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,197 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd08b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5e1b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59639c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:08,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:08,206 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e8d071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:08,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bc4caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:08,216 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,216 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,216 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,216 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,217 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,217 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,225 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T12:23:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:08,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,241 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,262 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:08,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:23:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:08,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:08,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:08,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:08,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/86a0c8b449b64cbfb82ce32cfe29af3d is 50, key is test_row_0/A:col10/1730982188251/Put/seqid=0 2024-11-07T12:23:08,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741837_1013 (size=12001) 2024-11-07T12:23:08,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/86a0c8b449b64cbfb82ce32cfe29af3d 2024-11-07T12:23:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:08,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:08,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:08,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982248579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982248564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982248585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982248589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982248593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2117d0c472d947dc9e25c92965333f64 is 50, key is test_row_0/B:col10/1730982188251/Put/seqid=0 2024-11-07T12:23:08,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741838_1014 (size=12001) 2024-11-07T12:23:08,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982248752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982248753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982248754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982248753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982248754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:08,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:08,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982248961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982248973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982248973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982248974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982248973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2117d0c472d947dc9e25c92965333f64 2024-11-07T12:23:09,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e8df4381fca94c799b633e7b7083d13e is 50, key is test_row_0/C:col10/1730982188251/Put/seqid=0 2024-11-07T12:23:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741839_1015 (size=12001) 2024-11-07T12:23:09,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982249271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982249293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982249293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982249292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982249302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:09,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:09,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:09,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:09,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:09,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e8df4381fca94c799b633e7b7083d13e 2024-11-07T12:23:09,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/86a0c8b449b64cbfb82ce32cfe29af3d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d 2024-11-07T12:23:09,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:23:09,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2117d0c472d947dc9e25c92965333f64 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64 2024-11-07T12:23:09,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:23:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e8df4381fca94c799b633e7b7083d13e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e 2024-11-07T12:23:09,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:23:09,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 533db45226dd865e0c0b2154b214ffef in 1321ms, sequenceid=12, compaction requested=false 2024-11-07T12:23:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:23:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:09,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T12:23:09,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:09,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:09,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e775fcc57a544038a311e735452cae31 is 50, key is test_row_0/A:col10/1730982188570/Put/seqid=0 2024-11-07T12:23:09,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741840_1016 (size=12001) 2024-11-07T12:23:09,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:09,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982249823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982249824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982249824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982249831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982249841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982249952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982249952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982249952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982249963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982250161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982250161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982250171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982250171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,183 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e775fcc57a544038a311e735452cae31 2024-11-07T12:23:10,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/34be93cc666d47f7844a4618ce90b43a is 50, key is test_row_0/B:col10/1730982188570/Put/seqid=0 2024-11-07T12:23:10,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741841_1017 (size=12001) 2024-11-07T12:23:10,273 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/34be93cc666d47f7844a4618ce90b43a 2024-11-07T12:23:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/f77274fb4eb24dc995fc73931fffe062 is 50, key is test_row_0/C:col10/1730982188570/Put/seqid=0 2024-11-07T12:23:10,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741842_1018 (size=12001) 2024-11-07T12:23:10,352 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/f77274fb4eb24dc995fc73931fffe062 2024-11-07T12:23:10,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e775fcc57a544038a311e735452cae31 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31 2024-11-07T12:23:10,390 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:23:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/34be93cc666d47f7844a4618ce90b43a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a 2024-11-07T12:23:10,408 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:23:10,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/f77274fb4eb24dc995fc73931fffe062 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062 2024-11-07T12:23:10,427 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T12:23:10,429 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 533db45226dd865e0c0b2154b214ffef in 711ms, sequenceid=38, compaction requested=false 2024-11-07T12:23:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T12:23:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T12:23:10,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T12:23:10,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-07T12:23:10,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2160 sec 2024-11-07T12:23:10,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:10,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:10,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16e8525782364b31b6dad3b0ae3ba3b6 is 50, key is test_row_0/A:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:10,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741843_1019 (size=12001) 2024-11-07T12:23:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982250618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982250623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982250623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982250623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982250743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982250743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982250742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982250743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982250852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,865 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 533db45226dd865e0c0b2154b214ffef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16e8525782364b31b6dad3b0ae3ba3b6 2024-11-07T12:23:10,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982250962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982250962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982250962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982250966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:10,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/13041a1baca449909a8711cd7e413e0a is 50, key is test_row_0/B:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:11,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741844_1020 (size=12001) 2024-11-07T12:23:11,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/13041a1baca449909a8711cd7e413e0a 2024-11-07T12:23:11,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e01906c429b94d1782cbb89d9fac66ea is 50, key is test_row_0/C:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741845_1021 (size=12001) 2024-11-07T12:23:11,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e01906c429b94d1782cbb89d9fac66ea 2024-11-07T12:23:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16e8525782364b31b6dad3b0ae3ba3b6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6 2024-11-07T12:23:11,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:23:11,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/13041a1baca449909a8711cd7e413e0a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a 2024-11-07T12:23:11,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:23:11,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e01906c429b94d1782cbb89d9fac66ea as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea 2024-11-07T12:23:11,157 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T12:23:11,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:23:11,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 533db45226dd865e0c0b2154b214ffef in 670ms, sequenceid=50, compaction requested=true 2024-11-07T12:23:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:11,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:11,185 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:11,185 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:11,190 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:11,191 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:11,192 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:11,192 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.2 K 2024-11-07T12:23:11,194 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2117d0c472d947dc9e25c92965333f64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982188251 2024-11-07T12:23:11,195 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:11,195 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:11,195 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:11,195 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.2 K 2024-11-07T12:23:11,197 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 34be93cc666d47f7844a4618ce90b43a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730982188570 2024-11-07T12:23:11,198 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 13041a1baca449909a8711cd7e413e0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:11,199 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a0c8b449b64cbfb82ce32cfe29af3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982188251 2024-11-07T12:23:11,201 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e775fcc57a544038a311e735452cae31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730982188570 2024-11-07T12:23:11,202 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e8525782364b31b6dad3b0ae3ba3b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:11,259 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:11,261 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/c8ee942a8b704652bf344d354c6c597d is 50, key is test_row_0/A:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:11,261 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#9 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:11,262 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/3bf6dd289fba46fabdbf55469b8bc32d is 50, key is test_row_0/B:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:11,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:11,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741846_1022 (size=12104) 2024-11-07T12:23:11,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741847_1023 (size=12104) 2024-11-07T12:23:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982251316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982251317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982251320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982251321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/094d5550576f4332bee8daf62bc24e13 is 50, key is test_row_0/A:col10/1730982191287/Put/seqid=0 2024-11-07T12:23:11,333 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/3bf6dd289fba46fabdbf55469b8bc32d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3bf6dd289fba46fabdbf55469b8bc32d 2024-11-07T12:23:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,342 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/c8ee942a8b704652bf344d354c6c597d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c8ee942a8b704652bf344d354c6c597d 2024-11-07T12:23:11,365 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 3bf6dd289fba46fabdbf55469b8bc32d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:11,365 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:11,370 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982191185; duration=0sec 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:11,371 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into c8ee942a8b704652bf344d354c6c597d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:11,371 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982191162; duration=0sec 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:11,371 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:11,374 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:11,374 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:11,374 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:11,374 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.2 K 2024-11-07T12:23:11,375 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e8df4381fca94c799b633e7b7083d13e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982188251 2024-11-07T12:23:11,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741848_1024 (size=14341) 2024-11-07T12:23:11,379 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f77274fb4eb24dc995fc73931fffe062, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730982188570 2024-11-07T12:23:11,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/094d5550576f4332bee8daf62bc24e13 2024-11-07T12:23:11,382 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e01906c429b94d1782cbb89d9fac66ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:11,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/9c059a3e77214993aca502a31314563f is 50, key is test_row_0/B:col10/1730982191287/Put/seqid=0 2024-11-07T12:23:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982251431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982251431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982251441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,448 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:11,449 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/8b4d9792bc684efb91ae2747e1e2b25c is 50, key is test_row_0/C:col10/1730982190486/Put/seqid=0 2024-11-07T12:23:11,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982251442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741849_1025 (size=12001) 2024-11-07T12:23:11,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741850_1026 (size=12104) 2024-11-07T12:23:11,509 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/8b4d9792bc684efb91ae2747e1e2b25c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/8b4d9792bc684efb91ae2747e1e2b25c 2024-11-07T12:23:11,526 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 8b4d9792bc684efb91ae2747e1e2b25c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:11,526 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:11,526 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982191185; duration=0sec 2024-11-07T12:23:11,527 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:11,527 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982251651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982251661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/9c059a3e77214993aca502a31314563f 2024-11-07T12:23:11,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b0400d4008824cf28b723a6f9d0eb6b3 is 50, key is test_row_0/C:col10/1730982191287/Put/seqid=0 2024-11-07T12:23:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741851_1027 (size=12001) 2024-11-07T12:23:11,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b0400d4008824cf28b723a6f9d0eb6b3 2024-11-07T12:23:11,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/094d5550576f4332bee8daf62bc24e13 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13 2024-11-07T12:23:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982251951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13, entries=200, sequenceid=77, filesize=14.0 K 2024-11-07T12:23:11,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982251961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982251961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/9c059a3e77214993aca502a31314563f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f 2024-11-07T12:23:11,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982251971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:11,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T12:23:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:11,983 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-07T12:23:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:11,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b0400d4008824cf28b723a6f9d0eb6b3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3 2024-11-07T12:23:12,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T12:23:12,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 533db45226dd865e0c0b2154b214ffef in 714ms, sequenceid=77, compaction requested=false 2024-11-07T12:23:12,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:12,048 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:23:12,117 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T12:23:12,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T12:23:12,119 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:23:12,119 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T12:23:12,120 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T12:23:12,120 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T12:23:12,121 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:23:12,121 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T12:23:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T12:23:12,402 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:12,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T12:23:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:23:12,414 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:12,417 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:12,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:12,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:12,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d3512ca98e1345da891414e049b96633 is 50, key is test_row_0/A:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741852_1028 (size=12001) 2024-11-07T12:23:12,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d3512ca98e1345da891414e049b96633 2024-11-07T12:23:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:23:12,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/37ff839b67bf4a34abf08d9cf8153b4e is 50, key is test_row_0/B:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982252537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982252539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982252541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982252542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741853_1029 (size=12001) 2024-11-07T12:23:12,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/37ff839b67bf4a34abf08d9cf8153b4e 2024-11-07T12:23:12,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:23:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:12,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a7332e797e654da1a3ed9d773259b5b1 is 50, key is test_row_0/C:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741854_1030 (size=12001) 2024-11-07T12:23:12,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a7332e797e654da1a3ed9d773259b5b1 2024-11-07T12:23:12,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d3512ca98e1345da891414e049b96633 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633 2024-11-07T12:23:12,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:23:12,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/37ff839b67bf4a34abf08d9cf8153b4e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e 2024-11-07T12:23:12,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982252651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982252651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982252652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982252652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:23:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a7332e797e654da1a3ed9d773259b5b1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1 2024-11-07T12:23:12,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:23:12,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 533db45226dd865e0c0b2154b214ffef in 217ms, sequenceid=91, compaction requested=true 2024-11-07T12:23:12,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:12,691 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:12,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:12,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:12,692 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:12,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:12,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:12,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:12,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:12,694 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:12,694 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:12,694 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:12,694 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:12,694 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,694 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,694 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c8ee942a8b704652bf344d354c6c597d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=37.5 K 2024-11-07T12:23:12,694 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3bf6dd289fba46fabdbf55469b8bc32d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.3 K 2024-11-07T12:23:12,695 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf6dd289fba46fabdbf55469b8bc32d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:12,695 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ee942a8b704652bf344d354c6c597d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:12,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c059a3e77214993aca502a31314563f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730982190616 2024-11-07T12:23:12,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 094d5550576f4332bee8daf62bc24e13, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730982190616 2024-11-07T12:23:12,697 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37ff839b67bf4a34abf08d9cf8153b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:12,698 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d3512ca98e1345da891414e049b96633, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:12,717 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:12,718 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2abb8ee9fa5444e4baa830edaecf4d19 is 50, key is test_row_0/A:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,725 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:23:12,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T12:23:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:12,741 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/a74a19f9a4364682a34209ad6f5158b1 is 50, key is test_row_0/B:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/93b02a01018d44e6831361d87e4448a2 is 50, key is test_row_0/A:col10/1730982192536/Put/seqid=0 2024-11-07T12:23:12,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741855_1031 (size=12207) 2024-11-07T12:23:12,767 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2abb8ee9fa5444e4baa830edaecf4d19 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2abb8ee9fa5444e4baa830edaecf4d19 2024-11-07T12:23:12,782 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 2abb8ee9fa5444e4baa830edaecf4d19(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:12,782 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:12,782 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982192691; duration=0sec 2024-11-07T12:23:12,782 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:12,782 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:12,783 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741856_1032 (size=12207) 2024-11-07T12:23:12,786 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:12,787 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:12,787 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:12,787 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/8b4d9792bc684efb91ae2747e1e2b25c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.3 K 2024-11-07T12:23:12,788 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4d9792bc684efb91ae2747e1e2b25c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982189811 2024-11-07T12:23:12,788 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b0400d4008824cf28b723a6f9d0eb6b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730982190616 2024-11-07T12:23:12,789 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a7332e797e654da1a3ed9d773259b5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:12,803 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/a74a19f9a4364682a34209ad6f5158b1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a74a19f9a4364682a34209ad6f5158b1 2024-11-07T12:23:12,816 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into a74a19f9a4364682a34209ad6f5158b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:12,816 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:12,816 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982192691; duration=0sec 2024-11-07T12:23:12,817 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:12,817 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:12,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741857_1033 (size=12001) 2024-11-07T12:23:12,822 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:12,822 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/93b02a01018d44e6831361d87e4448a2 2024-11-07T12:23:12,827 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/3b3d5833888141bbb10e79c7562b02d4 is 50, key is test_row_0/C:col10/1730982191316/Put/seqid=0 2024-11-07T12:23:12,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/a9484b9073c541aca8711f16daa25820 is 50, key is test_row_0/B:col10/1730982192536/Put/seqid=0 2024-11-07T12:23:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741858_1034 (size=12207) 2024-11-07T12:23:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:12,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:12,881 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/3b3d5833888141bbb10e79c7562b02d4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3b3d5833888141bbb10e79c7562b02d4 2024-11-07T12:23:12,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741859_1035 (size=12001) 2024-11-07T12:23:12,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982252879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982252877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/a9484b9073c541aca8711f16daa25820 2024-11-07T12:23:12,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982252889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982252889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982252891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,900 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 533db45226dd865e0c0b2154b214ffef of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4337 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:12,910 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 3b3d5833888141bbb10e79c7562b02d4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:12,910 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:12,910 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982192692; duration=0sec 2024-11-07T12:23:12,910 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:12,911 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:12,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a740119d818a495f852ee81ca651f80e is 50, key is test_row_0/C:col10/1730982192536/Put/seqid=0 2024-11-07T12:23:12,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741860_1036 (size=12001) 2024-11-07T12:23:12,952 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a740119d818a495f852ee81ca651f80e 2024-11-07T12:23:12,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/93b02a01018d44e6831361d87e4448a2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2 2024-11-07T12:23:12,981 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T12:23:12,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/a9484b9073c541aca8711f16daa25820 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820 2024-11-07T12:23:12,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982252991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982253001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,003 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T12:23:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982253001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/a740119d818a495f852ee81ca651f80e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e 2024-11-07T12:23:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982253001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T12:23:13,026 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 533db45226dd865e0c0b2154b214ffef in 288ms, sequenceid=116, compaction requested=false 2024-11-07T12:23:13,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:13,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:13,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T12:23:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T12:23:13,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T12:23:13,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 611 msec 2024-11-07T12:23:13,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 632 msec 2024-11-07T12:23:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T12:23:13,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:13,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T12:23:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:13,045 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:13,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:13,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:13,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T12:23:13,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:13,201 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:23:13,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:13,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/7f938ef91f7241c6af772732140dc09e is 50, key is test_row_0/A:col10/1730982192872/Put/seqid=0 2024-11-07T12:23:13,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982253264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982253266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982253269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741861_1037 (size=12051) 2024-11-07T12:23:13,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982253271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:13,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982253382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982253382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982253381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982253391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982253591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982253591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982253592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982253601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:13,677 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/7f938ef91f7241c6af772732140dc09e 2024-11-07T12:23:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/787e762b9d5348a383fbdc7bcd913dcf is 50, key is test_row_0/B:col10/1730982192872/Put/seqid=0 2024-11-07T12:23:13,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741862_1038 (size=12051) 2024-11-07T12:23:13,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982253901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982253902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982253902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:13,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982253911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,138 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/787e762b9d5348a383fbdc7bcd913dcf 2024-11-07T12:23:14,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/242d1f5dcd0745c080881f647761dca5 is 50, key is test_row_0/C:col10/1730982192872/Put/seqid=0 2024-11-07T12:23:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:14,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741863_1039 (size=12051) 2024-11-07T12:23:14,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/242d1f5dcd0745c080881f647761dca5 2024-11-07T12:23:14,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/7f938ef91f7241c6af772732140dc09e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e 2024-11-07T12:23:14,229 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T12:23:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/787e762b9d5348a383fbdc7bcd913dcf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf 2024-11-07T12:23:14,249 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T12:23:14,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/242d1f5dcd0745c080881f647761dca5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5 2024-11-07T12:23:14,265 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T12:23:14,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 533db45226dd865e0c0b2154b214ffef in 1066ms, sequenceid=131, compaction requested=true 2024-11-07T12:23:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T12:23:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T12:23:14,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T12:23:14,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-07T12:23:14,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2350 sec 2024-11-07T12:23:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:14,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:14,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16a543e45f364462a7d6dfa8e12c4dc1 is 50, key is test_row_0/A:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982254447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982254446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982254449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982254457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741864_1040 (size=16931) 2024-11-07T12:23:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982254561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982254562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982254561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982254571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982254772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982254772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982254781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982254781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:14,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16a543e45f364462a7d6dfa8e12c4dc1 2024-11-07T12:23:14,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/598903122f0b4f31928938fc3f844c79 is 50, key is test_row_0/B:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:14,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741865_1041 (size=12151) 2024-11-07T12:23:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982255081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982255081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982255090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982255091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T12:23:15,191 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:15,191 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T12:23:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:15,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:15,196 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:15,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:15,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/598903122f0b4f31928938fc3f844c79 2024-11-07T12:23:15,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/17cc45c6363d45d092cffa1196b75ab1 is 50, key is test_row_0/C:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741866_1042 (size=12151) 2024-11-07T12:23:15,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:23:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:23:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:15,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982255591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982255591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982255601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982255610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:15,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:23:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:15,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/17cc45c6363d45d092cffa1196b75ab1 2024-11-07T12:23:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/16a543e45f364462a7d6dfa8e12c4dc1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1 2024-11-07T12:23:15,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1, entries=250, sequenceid=157, filesize=16.5 K 2024-11-07T12:23:15,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/598903122f0b4f31928938fc3f844c79 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79 2024-11-07T12:23:15,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79, entries=150, sequenceid=157, filesize=11.9 K 2024-11-07T12:23:15,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/17cc45c6363d45d092cffa1196b75ab1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1 2024-11-07T12:23:15,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1, entries=150, sequenceid=157, filesize=11.9 K 2024-11-07T12:23:15,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 533db45226dd865e0c0b2154b214ffef in 1370ms, sequenceid=157, compaction requested=true 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:15,791 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:15,791 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:15,794 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:15,794 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:15,794 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,794 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a74a19f9a4364682a34209ad6f5158b1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=47.3 K 2024-11-07T12:23:15,795 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:15,795 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:15,795 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,795 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2abb8ee9fa5444e4baa830edaecf4d19, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=51.9 K 2024-11-07T12:23:15,796 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2abb8ee9fa5444e4baa830edaecf4d19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:15,796 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a74a19f9a4364682a34209ad6f5158b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:15,797 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b02a01018d44e6831361d87e4448a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730982192511 2024-11-07T12:23:15,797 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a9484b9073c541aca8711f16daa25820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730982192511 2024-11-07T12:23:15,798 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f938ef91f7241c6af772732140dc09e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982192872 2024-11-07T12:23:15,798 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 787e762b9d5348a383fbdc7bcd913dcf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982192872 2024-11-07T12:23:15,798 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a543e45f364462a7d6dfa8e12c4dc1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193257 2024-11-07T12:23:15,799 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 598903122f0b4f31928938fc3f844c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193265 2024-11-07T12:23:15,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T12:23:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,815 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:23:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:15,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/1118870ef95e47a1849c4a5b894e901c is 50, key is test_row_0/A:col10/1730982194431/Put/seqid=0 2024-11-07T12:23:15,830 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:15,830 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d30d89a8ff874df886f0e1533d0b9c3a is 50, key is test_row_0/A:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:15,834 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:15,834 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/638bee12faaa4706b82171aa18a66044 is 50, key is test_row_0/B:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741867_1043 (size=12151) 2024-11-07T12:23:15,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741868_1044 (size=12493) 2024-11-07T12:23:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741869_1045 (size=12493) 2024-11-07T12:23:15,866 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/638bee12faaa4706b82171aa18a66044 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/638bee12faaa4706b82171aa18a66044 2024-11-07T12:23:15,881 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 638bee12faaa4706b82171aa18a66044(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:15,881 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:15,881 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=12, startTime=1730982195791; duration=0sec 2024-11-07T12:23:15,882 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:15,882 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:15,882 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:15,892 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:15,892 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:15,892 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:15,893 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3b3d5833888141bbb10e79c7562b02d4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=47.3 K 2024-11-07T12:23:15,894 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b3d5833888141bbb10e79c7562b02d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982191313 2024-11-07T12:23:15,894 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a740119d818a495f852ee81ca651f80e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730982192511 2024-11-07T12:23:15,895 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 242d1f5dcd0745c080881f647761dca5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982192872 2024-11-07T12:23:15,895 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cc45c6363d45d092cffa1196b75ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193265 2024-11-07T12:23:15,927 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:15,929 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/c857da2e4cd64977a34a329a836563c9 is 50, key is test_row_0/C:col10/1730982193265/Put/seqid=0 2024-11-07T12:23:15,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741870_1046 (size=12493) 2024-11-07T12:23:15,989 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/c857da2e4cd64977a34a329a836563c9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c857da2e4cd64977a34a329a836563c9 2024-11-07T12:23:16,013 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into c857da2e4cd64977a34a329a836563c9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:16,013 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:16,013 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=12, startTime=1730982195791; duration=0sec 2024-11-07T12:23:16,013 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:16,017 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:16,247 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/1118870ef95e47a1849c4a5b894e901c 2024-11-07T12:23:16,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/c4d01a7603214506b9685808db3a37f7 is 50, key is test_row_0/B:col10/1730982194431/Put/seqid=0 2024-11-07T12:23:16,270 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d30d89a8ff874df886f0e1533d0b9c3a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d30d89a8ff874df886f0e1533d0b9c3a 2024-11-07T12:23:16,284 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into d30d89a8ff874df886f0e1533d0b9c3a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:16,285 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:16,285 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=12, startTime=1730982195791; duration=0sec 2024-11-07T12:23:16,285 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:16,285 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741871_1047 (size=12151) 2024-11-07T12:23:16,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/c4d01a7603214506b9685808db3a37f7 2024-11-07T12:23:16,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e6383d77c8b84094ac0b0272436642aa is 50, key is test_row_0/C:col10/1730982194431/Put/seqid=0 2024-11-07T12:23:16,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741872_1048 (size=12151) 2024-11-07T12:23:16,329 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e6383d77c8b84094ac0b0272436642aa 2024-11-07T12:23:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/1118870ef95e47a1849c4a5b894e901c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c 2024-11-07T12:23:16,356 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T12:23:16,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/c4d01a7603214506b9685808db3a37f7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7 2024-11-07T12:23:16,368 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T12:23:16,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/e6383d77c8b84094ac0b0272436642aa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa 2024-11-07T12:23:16,399 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T12:23:16,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 533db45226dd865e0c0b2154b214ffef in 585ms, sequenceid=167, compaction requested=false 2024-11-07T12:23:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T12:23:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T12:23:16,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T12:23:16,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-07T12:23:16,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2250 sec 2024-11-07T12:23:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:16,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:16,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/065435cdd5d2454d8cc65da37d7c9ccb is 50, key is test_row_0/A:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:16,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741873_1049 (size=12151) 2024-11-07T12:23:16,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/065435cdd5d2454d8cc65da37d7c9ccb 2024-11-07T12:23:16,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/acbb5ab726a84732b85271a99389488e is 50, key is test_row_0/B:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982256699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982256697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982256701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982256703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741874_1050 (size=12151) 2024-11-07T12:23:16,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/acbb5ab726a84732b85271a99389488e 2024-11-07T12:23:16,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4fc023e8a86c41148c984cc37e006514 is 50, key is test_row_0/C:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:16,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741875_1051 (size=12151) 2024-11-07T12:23:16,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982256811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982256811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982256811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982256811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982256930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:16,933 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 533db45226dd865e0c0b2154b214ffef of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8370 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:16,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982257022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982257022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982257022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982257022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4fc023e8a86c41148c984cc37e006514 2024-11-07T12:23:17,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/065435cdd5d2454d8cc65da37d7c9ccb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb 2024-11-07T12:23:17,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb, entries=150, sequenceid=182, filesize=11.9 K 2024-11-07T12:23:17,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/acbb5ab726a84732b85271a99389488e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e 2024-11-07T12:23:17,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e, entries=150, sequenceid=182, filesize=11.9 K 2024-11-07T12:23:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4fc023e8a86c41148c984cc37e006514 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514 2024-11-07T12:23:17,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514, entries=150, sequenceid=182, filesize=11.9 K 2024-11-07T12:23:17,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 533db45226dd865e0c0b2154b214ffef in 617ms, sequenceid=182, compaction requested=true 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:17,255 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:17,255 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:17,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:17,256 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:17,256 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:17,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:17,257 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,257 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,257 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/638bee12faaa4706b82171aa18a66044, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.9 K 2024-11-07T12:23:17,257 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d30d89a8ff874df886f0e1533d0b9c3a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.9 K 2024-11-07T12:23:17,257 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638bee12faaa4706b82171aa18a66044, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193265 2024-11-07T12:23:17,259 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d01a7603214506b9685808db3a37f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730982194431 2024-11-07T12:23:17,259 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d30d89a8ff874df886f0e1533d0b9c3a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193265 2024-11-07T12:23:17,259 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1118870ef95e47a1849c4a5b894e901c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730982194431 2024-11-07T12:23:17,259 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting acbb5ab726a84732b85271a99389488e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:17,260 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 065435cdd5d2454d8cc65da37d7c9ccb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:17,272 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:17,273 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/6794dfcd78234b129c8f91a9f697d147 is 50, key is test_row_0/B:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:17,289 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:17,290 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/0691f6754bab4c27aa63bb0df0d52197 is 50, key is test_row_0/A:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741877_1053 (size=12595) 2024-11-07T12:23:17,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741876_1052 (size=12595) 2024-11-07T12:23:17,333 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/0691f6754bab4c27aa63bb0df0d52197 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0691f6754bab4c27aa63bb0df0d52197 2024-11-07T12:23:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:17,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:17,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T12:23:17,341 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/6794dfcd78234b129c8f91a9f697d147 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6794dfcd78234b129c8f91a9f697d147 2024-11-07T12:23:17,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:17,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T12:23:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:17,345 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:17,346 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 0691f6754bab4c27aa63bb0df0d52197(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:17,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:17,346 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:17,347 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982197255; duration=0sec 2024-11-07T12:23:17,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:17,347 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:17,347 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:17,347 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:17,349 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:17,349 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:17,349 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,349 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c857da2e4cd64977a34a329a836563c9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=35.9 K 2024-11-07T12:23:17,350 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c857da2e4cd64977a34a329a836563c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730982193265 2024-11-07T12:23:17,351 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e6383d77c8b84094ac0b0272436642aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730982194431 2024-11-07T12:23:17,352 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc023e8a86c41148c984cc37e006514, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:17,357 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 6794dfcd78234b129c8f91a9f697d147(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:17,359 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:17,359 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982197255; duration=0sec 2024-11-07T12:23:17,360 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:17,360 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:17,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/c537bccd6a694638ad3bd56b12c8bafb is 50, key is test_row_0/A:col10/1730982197336/Put/seqid=0 2024-11-07T12:23:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982257359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982257359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982257371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982257373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,379 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:17,379 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/3f6fa00bb6644693b101788283fe761e is 50, key is test_row_0/C:col10/1730982196634/Put/seqid=0 2024-11-07T12:23:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741878_1054 (size=16931) 2024-11-07T12:23:17,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741879_1055 (size=12595) 2024-11-07T12:23:17,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/3f6fa00bb6644693b101788283fe761e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3f6fa00bb6644693b101788283fe761e 2024-11-07T12:23:17,435 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 3f6fa00bb6644693b101788283fe761e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:17,435 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:17,435 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982197255; duration=0sec 2024-11-07T12:23:17,435 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:17,435 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:17,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982257481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982257481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982257481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982257482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:17,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:17,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:17,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982257691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982257691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982257691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982257691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:17,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/c537bccd6a694638ad3bd56b12c8bafb 2024-11-07T12:23:17,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/399be3c3e74e4cf5b34b990afd31bd5b is 50, key is test_row_0/B:col10/1730982197336/Put/seqid=0 2024-11-07T12:23:17,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741880_1056 (size=12151) 2024-11-07T12:23:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:17,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:17,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:17,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:17,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982258001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982258001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982258001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982258001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/399be3c3e74e4cf5b34b990afd31bd5b 2024-11-07T12:23:18,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/5e94a36a5c084268a7979088ed21af7b is 50, key is test_row_0/C:col10/1730982197336/Put/seqid=0 2024-11-07T12:23:18,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741881_1057 (size=12151) 2024-11-07T12:23:18,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/5e94a36a5c084268a7979088ed21af7b 2024-11-07T12:23:18,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:18,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:18,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/c537bccd6a694638ad3bd56b12c8bafb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb 2024-11-07T12:23:18,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb, entries=250, sequenceid=208, filesize=16.5 K 2024-11-07T12:23:18,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/399be3c3e74e4cf5b34b990afd31bd5b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b 2024-11-07T12:23:18,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T12:23:18,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/5e94a36a5c084268a7979088ed21af7b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b 2024-11-07T12:23:18,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T12:23:18,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 533db45226dd865e0c0b2154b214ffef in 1001ms, sequenceid=208, compaction requested=false 2024-11-07T12:23:18,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:18,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T12:23:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,443 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:18,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/915ba3472dfd4e71bd647b42ca05960c is 50, key is test_row_0/A:col10/1730982197355/Put/seqid=0 2024-11-07T12:23:18,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741882_1058 (size=12151) 2024-11-07T12:23:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:18,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982258567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982258566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982258572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982258570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982258680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982258681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982258681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982258691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,859 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/915ba3472dfd4e71bd647b42ca05960c 2024-11-07T12:23:18,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2ff96b42dd4c4c09a0b8a81dced2c78a is 50, key is test_row_0/B:col10/1730982197355/Put/seqid=0 2024-11-07T12:23:18,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741883_1059 (size=12151) 2024-11-07T12:23:18,880 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2ff96b42dd4c4c09a0b8a81dced2c78a 2024-11-07T12:23:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4125660586254009868b71bfbd098546 is 50, key is test_row_0/C:col10/1730982197355/Put/seqid=0 2024-11-07T12:23:18,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982258891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982258891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982258891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982258902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:18,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741884_1060 (size=12151) 2024-11-07T12:23:18,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4125660586254009868b71bfbd098546 2024-11-07T12:23:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/915ba3472dfd4e71bd647b42ca05960c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c 2024-11-07T12:23:18,931 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T12:23:18,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/2ff96b42dd4c4c09a0b8a81dced2c78a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a 2024-11-07T12:23:18,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T12:23:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/4125660586254009868b71bfbd098546 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546 2024-11-07T12:23:18,952 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T12:23:18,954 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 533db45226dd865e0c0b2154b214ffef in 510ms, sequenceid=222, compaction requested=true 2024-11-07T12:23:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T12:23:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T12:23:18,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T12:23:18,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-07T12:23:18,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6200 sec 2024-11-07T12:23:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:19,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:19,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T12:23:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982259214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982259216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d00c8b8069374e2fb1becbaff0b7718e is 50, key is test_row_0/A:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:19,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982259216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982259217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741885_1061 (size=19321) 2024-11-07T12:23:19,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d00c8b8069374e2fb1becbaff0b7718e 2024-11-07T12:23:19,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/62b9049cb06e4ef7ad28f803287530ad is 50, key is test_row_0/B:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:19,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741886_1062 (size=12151) 2024-11-07T12:23:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982259331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982259331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982259331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982259331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T12:23:19,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:19,491 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T12:23:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:19,495 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:19,496 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:19,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982259541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982259542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982259542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982259542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:19,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:23:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/62b9049cb06e4ef7ad28f803287530ad 2024-11-07T12:23:19,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/22b25aac8e724ea98e1afa8091ba214a is 50, key is test_row_0/C:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:19,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741887_1063 (size=12151) 2024-11-07T12:23:19,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:23:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982259850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982259851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982259851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982259851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:19,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:23:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:19,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:20,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/22b25aac8e724ea98e1afa8091ba214a 2024-11-07T12:23:20,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d00c8b8069374e2fb1becbaff0b7718e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e 2024-11-07T12:23:20,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e, entries=300, sequenceid=251, filesize=18.9 K 2024-11-07T12:23:20,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/62b9049cb06e4ef7ad28f803287530ad as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad 2024-11-07T12:23:20,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:23:20,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/22b25aac8e724ea98e1afa8091ba214a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a 2024-11-07T12:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:20,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:23:20,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:23:20,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:20,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 533db45226dd865e0c0b2154b214ffef in 916ms, sequenceid=251, compaction requested=true 2024-11-07T12:23:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:20,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:20,125 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:20,125 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:20,128 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60998 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:20,128 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:20,128 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:20,128 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,128 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:20,128 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,129 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0691f6754bab4c27aa63bb0df0d52197, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=59.6 K 2024-11-07T12:23:20,129 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6794dfcd78234b129c8f91a9f697d147, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=47.9 K 2024-11-07T12:23:20,129 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0691f6754bab4c27aa63bb0df0d52197, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:20,129 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6794dfcd78234b129c8f91a9f697d147, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:20,129 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c537bccd6a694638ad3bd56b12c8bafb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982196696 2024-11-07T12:23:20,129 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 399be3c3e74e4cf5b34b990afd31bd5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982196701 2024-11-07T12:23:20,130 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff96b42dd4c4c09a0b8a81dced2c78a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730982197352 2024-11-07T12:23:20,130 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 915ba3472dfd4e71bd647b42ca05960c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730982197352 2024-11-07T12:23:20,131 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d00c8b8069374e2fb1becbaff0b7718e, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:20,131 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b9049cb06e4ef7ad28f803287530ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:20,147 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:20,148 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/4100b5cb597448819541659716ba0c41 is 50, key is test_row_0/B:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:20,149 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#51 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:20,149 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/70fcb64f5844443db0d5f0b6b7779bb3 is 50, key is test_row_0/A:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:20,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741889_1065 (size=12731) 2024-11-07T12:23:20,211 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/70fcb64f5844443db0d5f0b6b7779bb3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/70fcb64f5844443db0d5f0b6b7779bb3 2024-11-07T12:23:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741888_1064 (size=12731) 2024-11-07T12:23:20,226 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 70fcb64f5844443db0d5f0b6b7779bb3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:20,226 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:20,226 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=12, startTime=1730982200124; duration=0sec 2024-11-07T12:23:20,226 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:20,226 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:20,226 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:20,229 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:20,229 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:20,229 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,229 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3f6fa00bb6644693b101788283fe761e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=47.9 K 2024-11-07T12:23:20,230 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6fa00bb6644693b101788283fe761e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730982196624 2024-11-07T12:23:20,230 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e94a36a5c084268a7979088ed21af7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982196701 2024-11-07T12:23:20,231 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4125660586254009868b71bfbd098546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730982197352 2024-11-07T12:23:20,232 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b25aac8e724ea98e1afa8091ba214a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:20,256 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#53 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:20,257 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/68ce989ac7384f308fd9a91c3a40344f is 50, key is test_row_0/C:col10/1730982198566/Put/seqid=0 2024-11-07T12:23:20,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T12:23:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T12:23:20,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741890_1066 (size=12731) 2024-11-07T12:23:20,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2fd929dd586b49d2bcb9b70adfc4f625 is 50, key is test_row_2/A:col10/1730982199215/Put/seqid=0 2024-11-07T12:23:20,305 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/68ce989ac7384f308fd9a91c3a40344f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68ce989ac7384f308fd9a91c3a40344f 2024-11-07T12:23:20,331 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 68ce989ac7384f308fd9a91c3a40344f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:20,331 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:20,331 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=12, startTime=1730982200125; duration=0sec 2024-11-07T12:23:20,331 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:20,332 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741891_1067 (size=7415) 2024-11-07T12:23:20,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:20,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982260415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982260426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982260427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982260427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982260532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982260542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982260545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982260545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,629 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/4100b5cb597448819541659716ba0c41 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/4100b5cb597448819541659716ba0c41 2024-11-07T12:23:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:20,651 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 4100b5cb597448819541659716ba0c41(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:20,651 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:20,651 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=12, startTime=1730982200125; duration=0sec 2024-11-07T12:23:20,652 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:20,652 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982260741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982260751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982260751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982260752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2fd929dd586b49d2bcb9b70adfc4f625 2024-11-07T12:23:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/e42a28bee625434bb76467ed33e773d1 is 50, key is test_row_2/B:col10/1730982199215/Put/seqid=0 2024-11-07T12:23:20,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741892_1068 (size=7415) 2024-11-07T12:23:20,818 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/e42a28bee625434bb76467ed33e773d1 2024-11-07T12:23:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/c458bb66b0ae41db82f79e1c95890c55 is 50, key is test_row_2/C:col10/1730982199215/Put/seqid=0 2024-11-07T12:23:20,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741893_1069 (size=7415) 2024-11-07T12:23:21,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982261060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982261061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982261061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982261061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/c458bb66b0ae41db82f79e1c95890c55 2024-11-07T12:23:21,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2fd929dd586b49d2bcb9b70adfc4f625 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625 2024-11-07T12:23:21,261 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625, entries=50, sequenceid=259, filesize=7.2 K 2024-11-07T12:23:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/e42a28bee625434bb76467ed33e773d1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1 2024-11-07T12:23:21,271 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1, entries=50, sequenceid=259, filesize=7.2 K 2024-11-07T12:23:21,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/c458bb66b0ae41db82f79e1c95890c55 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55 2024-11-07T12:23:21,284 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55, entries=50, sequenceid=259, filesize=7.2 K 2024-11-07T12:23:21,285 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for 533db45226dd865e0c0b2154b214ffef in 1004ms, sequenceid=259, compaction requested=false 2024-11-07T12:23:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T12:23:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T12:23:21,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T12:23:21,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-11-07T12:23:21,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8010 sec 2024-11-07T12:23:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:21,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-07T12:23:21,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982261576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982261613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982261614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982261618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/6d282149189a43ae956cdef8f1531862 is 50, key is test_row_0/A:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741894_1070 (size=12301) 2024-11-07T12:23:21,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/6d282149189a43ae956cdef8f1531862 2024-11-07T12:23:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T12:23:21,652 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:21,653 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T12:23:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:23:21,663 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:21,664 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:21,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:21,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/00f3a8aa93c04a83a715225aec6ce58b is 50, key is test_row_0/B:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:21,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741895_1071 (size=12301) 2024-11-07T12:23:21,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982261721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982261731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982261731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:23:21,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:21,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982261930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982261941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982261941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:21,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:21,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:21,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:21,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:21,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:21,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:23:22,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/00f3a8aa93c04a83a715225aec6ce58b 2024-11-07T12:23:22,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/7b476ebd970f4e468208420ca5446cce is 50, key is test_row_0/C:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:22,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741896_1072 (size=12301) 2024-11-07T12:23:22,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982262241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982262251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982262251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:22,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:23:22,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/7b476ebd970f4e468208420ca5446cce 2024-11-07T12:23:22,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/6d282149189a43ae956cdef8f1531862 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862 2024-11-07T12:23:22,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:23:22,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/00f3a8aa93c04a83a715225aec6ce58b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b 2024-11-07T12:23:22,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:23:22,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/7b476ebd970f4e468208420ca5446cce as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce 2024-11-07T12:23:22,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:23:22,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 533db45226dd865e0c0b2154b214ffef in 944ms, sequenceid=294, compaction requested=true 2024-11-07T12:23:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:22,557 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:22,557 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:22,558 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:22,558 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:22,559 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,559 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/70fcb64f5844443db0d5f0b6b7779bb3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=31.7 K 2024-11-07T12:23:22,559 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:22,559 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70fcb64f5844443db0d5f0b6b7779bb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:22,559 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:22,559 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,560 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/4100b5cb597448819541659716ba0c41, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=31.7 K 2024-11-07T12:23:22,560 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fd929dd586b49d2bcb9b70adfc4f625, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1730982199215 2024-11-07T12:23:22,560 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4100b5cb597448819541659716ba0c41, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:22,561 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d282149189a43ae956cdef8f1531862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:22,561 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e42a28bee625434bb76467ed33e773d1, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1730982199215 2024-11-07T12:23:22,561 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f3a8aa93c04a83a715225aec6ce58b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:22,571 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#61 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:22,572 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:22,572 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/129550d89d574457930e47c0316226f4 is 50, key is test_row_0/A:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:22,572 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/891109e808e848dcac5e6d5aed56eab3 is 50, key is test_row_0/B:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:22,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741898_1074 (size=12983) 2024-11-07T12:23:22,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T12:23:22,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:22,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T12:23:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T12:23:22,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741897_1073 (size=12983) 2024-11-07T12:23:22,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T12:23:22,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 932 msec 2024-11-07T12:23:22,605 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/129550d89d574457930e47c0316226f4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/129550d89d574457930e47c0316226f4 2024-11-07T12:23:22,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 951 msec 2024-11-07T12:23:22,607 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/891109e808e848dcac5e6d5aed56eab3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/891109e808e848dcac5e6d5aed56eab3 2024-11-07T12:23:22,615 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 129550d89d574457930e47c0316226f4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:22,616 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:22,616 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982202557; duration=0sec 2024-11-07T12:23:22,616 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:22,616 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:22,616 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:22,617 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 891109e808e848dcac5e6d5aed56eab3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:22,617 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:22,617 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:22,617 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:22,618 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982202557; duration=0sec 2024-11-07T12:23:22,618 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68ce989ac7384f308fd9a91c3a40344f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=31.7 K 2024-11-07T12:23:22,618 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ce989ac7384f308fd9a91c3a40344f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982198553 2024-11-07T12:23:22,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:22,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:22,619 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c458bb66b0ae41db82f79e1c95890c55, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1730982199215 2024-11-07T12:23:22,619 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b476ebd970f4e468208420ca5446cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:22,628 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:22,628 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/db8e5feaaaaa4038a31c6b0add349e9e is 50, key is test_row_0/C:col10/1730982201575/Put/seqid=0 2024-11-07T12:23:22,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741899_1075 (size=12983) 2024-11-07T12:23:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:22,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:22,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:22,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d223705a4c1a459587be1581861d0409 is 50, key is test_row_0/A:col10/1730982202647/Put/seqid=0 2024-11-07T12:23:22,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741900_1076 (size=12301) 2024-11-07T12:23:22,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d223705a4c1a459587be1581861d0409 2024-11-07T12:23:22,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/583204f998a149db9151c45f646e1443 is 50, key is test_row_0/B:col10/1730982202647/Put/seqid=0 2024-11-07T12:23:22,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741901_1077 (size=12301) 2024-11-07T12:23:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982262735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982262751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982262761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982262761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T12:23:22,801 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:22,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T12:23:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:22,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:22,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:22,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:22,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982262840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:22,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:23:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:22,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,046 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/db8e5feaaaaa4038a31c6b0add349e9e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/db8e5feaaaaa4038a31c6b0add349e9e 2024-11-07T12:23:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982263051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,057 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into db8e5feaaaaa4038a31c6b0add349e9e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:23,057 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:23,057 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982202558; duration=0sec 2024-11-07T12:23:23,057 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:23,057 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:23,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/583204f998a149db9151c45f646e1443 2024-11-07T12:23:23,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/650e5b9b07d44caeb523e92114a619bf is 50, key is test_row_0/C:col10/1730982202647/Put/seqid=0 2024-11-07T12:23:23,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:23:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741902_1078 (size=12301) 2024-11-07T12:23:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:23,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:23:23,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:23,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982263361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:23:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:23,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/650e5b9b07d44caeb523e92114a619bf 2024-11-07T12:23:23,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/d223705a4c1a459587be1581861d0409 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409 2024-11-07T12:23:23,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409, entries=150, sequenceid=308, filesize=12.0 K 2024-11-07T12:23:23,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/583204f998a149db9151c45f646e1443 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443 2024-11-07T12:23:23,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443, entries=150, sequenceid=308, filesize=12.0 K 2024-11-07T12:23:23,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/650e5b9b07d44caeb523e92114a619bf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf 2024-11-07T12:23:23,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf, entries=150, sequenceid=308, filesize=12.0 K 2024-11-07T12:23:23,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 533db45226dd865e0c0b2154b214ffef in 907ms, sequenceid=308, compaction requested=false 2024-11-07T12:23:23,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:23,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T12:23:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:23,578 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/5bff6723e35e4f5da035e2b884441d0e is 50, key is test_row_0/A:col10/1730982202723/Put/seqid=0 2024-11-07T12:23:23,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741903_1079 (size=12301) 2024-11-07T12:23:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:23,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982263787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982263788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982263789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982263871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982263900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982263901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982263900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:23,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/5bff6723e35e4f5da035e2b884441d0e 2024-11-07T12:23:24,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/3f26426a750b4a8b951f7ad15e162ca4 is 50, key is test_row_0/B:col10/1730982202723/Put/seqid=0 2024-11-07T12:23:24,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741904_1080 (size=12301) 2024-11-07T12:23:24,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/3f26426a750b4a8b951f7ad15e162ca4 2024-11-07T12:23:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/598697cb069146baaaa6e72861be9d18 is 50, key is test_row_0/C:col10/1730982202723/Put/seqid=0 2024-11-07T12:23:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741905_1081 (size=12301) 2024-11-07T12:23:24,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982264111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982264111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982264111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982264421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982264421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982264421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,456 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/598697cb069146baaaa6e72861be9d18 2024-11-07T12:23:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/5bff6723e35e4f5da035e2b884441d0e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e 2024-11-07T12:23:24,467 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T12:23:24,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/3f26426a750b4a8b951f7ad15e162ca4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4 2024-11-07T12:23:24,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T12:23:24,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/598697cb069146baaaa6e72861be9d18 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18 2024-11-07T12:23:24,479 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T12:23:24,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 533db45226dd865e0c0b2154b214ffef in 902ms, sequenceid=334, compaction requested=true 2024-11-07T12:23:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T12:23:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T12:23:24,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T12:23:24,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-07T12:23:24,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6820 sec 2024-11-07T12:23:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:24,882 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:24,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:23:24,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/796e7da57fce4a8f835faa6bfb490592 is 50, key is test_row_0/A:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741906_1082 (size=14741) 2024-11-07T12:23:24,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/796e7da57fce4a8f835faa6bfb490592 2024-11-07T12:23:24,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/951757aaced14211ab0c5c6b1494923f is 50, key is test_row_0/B:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741907_1083 (size=12301) 2024-11-07T12:23:24,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/951757aaced14211ab0c5c6b1494923f 2024-11-07T12:23:24,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/26cc8d24361d4bcebb45d69a185350f1 is 50, key is test_row_0/C:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:24,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741908_1084 (size=12301) 2024-11-07T12:23:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982264943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982264948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982264949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982264949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T12:23:24,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:24,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-07T12:23:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:23:24,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:24,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:24,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:25,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982265061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982265061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982265061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982265061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:23:25,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T12:23:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982265271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982265271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982265271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982265272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:23:25,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/26cc8d24361d4bcebb45d69a185350f1 2024-11-07T12:23:25,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/796e7da57fce4a8f835faa6bfb490592 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592 2024-11-07T12:23:25,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592, entries=200, sequenceid=346, filesize=14.4 K 2024-11-07T12:23:25,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/951757aaced14211ab0c5c6b1494923f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f 2024-11-07T12:23:25,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-07T12:23:25,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/26cc8d24361d4bcebb45d69a185350f1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1 2024-11-07T12:23:25,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1, entries=150, sequenceid=346, filesize=12.0 K 2024-11-07T12:23:25,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 533db45226dd865e0c0b2154b214ffef in 476ms, sequenceid=346, compaction requested=true 2024-11-07T12:23:25,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:25,359 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:25,359 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:25,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:25,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:25,361 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:25,361 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:25,361 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,362 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/129550d89d574457930e47c0316226f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=51.1 K 2024-11-07T12:23:25,362 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:25,362 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:25,362 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,363 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/891109e808e848dcac5e6d5aed56eab3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=48.7 K 2024-11-07T12:23:25,363 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 129550d89d574457930e47c0316226f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:25,363 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 891109e808e848dcac5e6d5aed56eab3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:25,363 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d223705a4c1a459587be1581861d0409, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1730982202637 2024-11-07T12:23:25,364 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 583204f998a149db9151c45f646e1443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1730982202637 2024-11-07T12:23:25,364 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bff6723e35e4f5da035e2b884441d0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1730982202712 2024-11-07T12:23:25,364 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f26426a750b4a8b951f7ad15e162ca4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1730982202712 2024-11-07T12:23:25,365 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 951757aaced14211ab0c5c6b1494923f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:25,365 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796e7da57fce4a8f835faa6bfb490592, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:25,379 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:25,379 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:25,380 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/ff5ba0ca58414356822366408625f0e6 is 50, key is test_row_0/A:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:25,380 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/9b185515239d4e3aa6b96238eabaf17a is 50, key is test_row_0/B:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:25,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741909_1085 (size=13119) 2024-11-07T12:23:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741910_1086 (size=13119) 2024-11-07T12:23:25,416 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/9b185515239d4e3aa6b96238eabaf17a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9b185515239d4e3aa6b96238eabaf17a 2024-11-07T12:23:25,423 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into 9b185515239d4e3aa6b96238eabaf17a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:25,423 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:25,423 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=12, startTime=1730982205359; duration=0sec 2024-11-07T12:23:25,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:25,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:25,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:25,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:25,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:25,426 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T12:23:25,426 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/db8e5feaaaaa4038a31c6b0add349e9e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=48.7 K 2024-11-07T12:23:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:25,427 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting db8e5feaaaaa4038a31c6b0add349e9e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982201572 2024-11-07T12:23:25,428 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 650e5b9b07d44caeb523e92114a619bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1730982202637 2024-11-07T12:23:25,429 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 598697cb069146baaaa6e72861be9d18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1730982202712 2024-11-07T12:23:25,430 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 26cc8d24361d4bcebb45d69a185350f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2f5410d595974e558a199b28509c7338 is 50, key is test_row_0/A:col10/1730982204934/Put/seqid=0 2024-11-07T12:23:25,445 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:25,445 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/7edbb641e3d34240a35385db9e8eb996 is 50, key is test_row_0/C:col10/1730982204881/Put/seqid=0 2024-11-07T12:23:25,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741911_1087 (size=12301) 2024-11-07T12:23:25,458 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2f5410d595974e558a199b28509c7338 2024-11-07T12:23:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/72bab7fbeb40425aa5fa857b6f843c53 is 50, key is test_row_0/B:col10/1730982204934/Put/seqid=0 2024-11-07T12:23:25,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741912_1088 (size=13119) 2024-11-07T12:23:25,488 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/7edbb641e3d34240a35385db9e8eb996 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7edbb641e3d34240a35385db9e8eb996 2024-11-07T12:23:25,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741913_1089 (size=12301) 2024-11-07T12:23:25,498 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 7edbb641e3d34240a35385db9e8eb996(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:25,499 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:25,499 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=12, startTime=1730982205360; duration=0sec 2024-11-07T12:23:25,499 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:25,499 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:23:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982265587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982265588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982265589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982265591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982265701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982265701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982265701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982265701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,805 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/ff5ba0ca58414356822366408625f0e6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/ff5ba0ca58414356822366408625f0e6 2024-11-07T12:23:25,812 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into ff5ba0ca58414356822366408625f0e6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:25,812 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:25,812 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=12, startTime=1730982205358; duration=0sec 2024-11-07T12:23:25,812 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:25,812 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:25,890 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/72bab7fbeb40425aa5fa857b6f843c53 2024-11-07T12:23:25,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/68bb9c62de494a09b1539a06cc7cbc06 is 50, key is test_row_0/C:col10/1730982204934/Put/seqid=0 2024-11-07T12:23:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741914_1090 (size=12301) 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/68bb9c62de494a09b1539a06cc7cbc06 2024-11-07T12:23:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982265911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:25,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2f5410d595974e558a199b28509c7338 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338 2024-11-07T12:23:25,927 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338, entries=150, sequenceid=372, filesize=12.0 K 2024-11-07T12:23:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/72bab7fbeb40425aa5fa857b6f843c53 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53 2024-11-07T12:23:25,934 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53, entries=150, sequenceid=372, filesize=12.0 K 2024-11-07T12:23:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/68bb9c62de494a09b1539a06cc7cbc06 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06 2024-11-07T12:23:25,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06, entries=150, sequenceid=372, filesize=12.0 K 2024-11-07T12:23:25,942 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 533db45226dd865e0c0b2154b214ffef in 515ms, sequenceid=372, compaction requested=false 2024-11-07T12:23:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-07T12:23:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-07T12:23:25,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T12:23:25,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 978 msec 2024-11-07T12:23:25,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 984 msec 2024-11-07T12:23:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T12:23:26,101 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:26,102 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-07T12:23:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:26,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:26,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:26,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:26,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:23:26,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/0ef22f9527644a51adb8d692fd2837e4 is 50, key is test_row_0/A:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982266248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741915_1091 (size=14741) 2024-11-07T12:23:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982266252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:26,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/0ef22f9527644a51adb8d692fd2837e4 2024-11-07T12:23:26,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982266253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982266255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/6b15e69b39994e64b93d7cf0ab7c7633 is 50, key is test_row_0/B:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:26,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741916_1092 (size=12301) 2024-11-07T12:23:26,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982266361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982266371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982266371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982266371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:26,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:26,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982266571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982266581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982266581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982266581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:26,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:26,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/6b15e69b39994e64b93d7cf0ab7c7633 2024-11-07T12:23:26,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/bb7ba570be33423a83641ddcc31f8b77 is 50, key is test_row_0/C:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:26,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741917_1093 (size=12301) 2024-11-07T12:23:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982266881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982266891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982266891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982266891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:26,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:26,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:27,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1730982267011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,012 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 533db45226dd865e0c0b2154b214ffef of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18449 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:27,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/bb7ba570be33423a83641ddcc31f8b77 2024-11-07T12:23:27,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/0ef22f9527644a51adb8d692fd2837e4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4 2024-11-07T12:23:27,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4, entries=200, sequenceid=388, filesize=14.4 K 2024-11-07T12:23:27,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/6b15e69b39994e64b93d7cf0ab7c7633 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633 2024-11-07T12:23:27,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633, entries=150, sequenceid=388, filesize=12.0 K 2024-11-07T12:23:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/bb7ba570be33423a83641ddcc31f8b77 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77 2024-11-07T12:23:27,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77, entries=150, sequenceid=388, filesize=12.0 K 2024-11-07T12:23:27,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 533db45226dd865e0c0b2154b214ffef in 969ms, sequenceid=388, compaction requested=true 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:27,195 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:27,195 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:27,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:27,196 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:27,196 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:27,196 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:27,196 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:27,197 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,197 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,197 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9b185515239d4e3aa6b96238eabaf17a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=36.8 K 2024-11-07T12:23:27,197 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/ff5ba0ca58414356822366408625f0e6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=39.2 K 2024-11-07T12:23:27,197 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b185515239d4e3aa6b96238eabaf17a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:27,197 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff5ba0ca58414356822366408625f0e6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:27,198 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5410d595974e558a199b28509c7338, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1730982204934 2024-11-07T12:23:27,198 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bab7fbeb40425aa5fa857b6f843c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1730982204934 2024-11-07T12:23:27,198 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef22f9527644a51adb8d692fd2837e4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:27,198 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b15e69b39994e64b93d7cf0ab7c7633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:27,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T12:23:27,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,207 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:23:27,214 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:27,215 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/c58cf1fc08de408ab4b3878ec0be4a34 is 50, key is test_row_0/B:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:27,218 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:27,219 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/158568a93236439eb45efbc6a8da6e7d is 50, key is test_row_0/A:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/318d5cf8e20d47388cce5cb38a238d01 is 50, key is test_row_0/A:col10/1730982206248/Put/seqid=0 2024-11-07T12:23:27,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741920_1096 (size=12301) 2024-11-07T12:23:27,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741918_1094 (size=13221) 2024-11-07T12:23:27,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741919_1095 (size=13221) 2024-11-07T12:23:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:27,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. as already flushing 2024-11-07T12:23:27,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49522 deadline: 1730982267411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49548 deadline: 1730982267411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982267413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982267414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982267521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982267521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,630 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/318d5cf8e20d47388cce5cb38a238d01 2024-11-07T12:23:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/098a836e1ca14678af9826bd9720f770 is 50, key is test_row_0/B:col10/1730982206248/Put/seqid=0 2024-11-07T12:23:27,641 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/c58cf1fc08de408ab4b3878ec0be4a34 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c58cf1fc08de408ab4b3878ec0be4a34 2024-11-07T12:23:27,646 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/158568a93236439eb45efbc6a8da6e7d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/158568a93236439eb45efbc6a8da6e7d 2024-11-07T12:23:27,649 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into c58cf1fc08de408ab4b3878ec0be4a34(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:27,649 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:27,649 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982207195; duration=0sec 2024-11-07T12:23:27,649 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:27,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:27,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741921_1097 (size=12301) 2024-11-07T12:23:27,652 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/098a836e1ca14678af9826bd9720f770 2024-11-07T12:23:27,652 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:27,652 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:27,652 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:27,653 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7edbb641e3d34240a35385db9e8eb996, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=36.8 K 2024-11-07T12:23:27,653 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 7edbb641e3d34240a35385db9e8eb996, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1730982203785 2024-11-07T12:23:27,654 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 68bb9c62de494a09b1539a06cc7cbc06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1730982204934 2024-11-07T12:23:27,654 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7ba570be33423a83641ddcc31f8b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:27,656 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 158568a93236439eb45efbc6a8da6e7d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:27,656 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:27,656 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982207195; duration=0sec 2024-11-07T12:23:27,656 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:27,656 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:27,671 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:27,672 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/101c2174e75d4dcd8a6d51f0a5084370 is 50, key is test_row_0/C:col10/1730982206224/Put/seqid=0 2024-11-07T12:23:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b92ada5670c74e38883fa9e005b51f9c is 50, key is test_row_0/C:col10/1730982206248/Put/seqid=0 2024-11-07T12:23:27,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741922_1098 (size=13221) 2024-11-07T12:23:27,692 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/101c2174e75d4dcd8a6d51f0a5084370 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/101c2174e75d4dcd8a6d51f0a5084370 2024-11-07T12:23:27,701 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 101c2174e75d4dcd8a6d51f0a5084370(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:27,701 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:27,701 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982207195; duration=0sec 2024-11-07T12:23:27,701 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:27,701 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:27,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741923_1099 (size=12301) 2024-11-07T12:23:27,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982267731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982267731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:27,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:27,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49544 deadline: 1730982268041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49530 deadline: 1730982268040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:23:28,107 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b92ada5670c74e38883fa9e005b51f9c 2024-11-07T12:23:28,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/318d5cf8e20d47388cce5cb38a238d01 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01 2024-11-07T12:23:28,124 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01, entries=150, sequenceid=411, filesize=12.0 K 2024-11-07T12:23:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/098a836e1ca14678af9826bd9720f770 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770 2024-11-07T12:23:28,142 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770, entries=150, sequenceid=411, filesize=12.0 K 2024-11-07T12:23:28,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/b92ada5670c74e38883fa9e005b51f9c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c 2024-11-07T12:23:28,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c, entries=150, sequenceid=411, filesize=12.0 K 2024-11-07T12:23:28,151 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 533db45226dd865e0c0b2154b214ffef in 944ms, sequenceid=411, compaction requested=false 2024-11-07T12:23:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-07T12:23:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-07T12:23:28,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T12:23:28,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0480 sec 2024-11-07T12:23:28,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0550 sec 2024-11-07T12:23:28,250 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T12:23:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T12:23:28,251 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:28,252 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T12:23:28,252 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,254 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T12:23:28,254 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,255 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T12:23:28,255 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,256 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,257 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:28,422 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:28,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:23:28,423 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T12:23:28,423 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T12:23:28,423 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,423 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,423 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,423 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,423 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,423 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/eeb4f1bd0ac848bba4b1033041220fa5 is 50, key is test_row_0/A:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741924_1100 (size=12301) 2024-11-07T12:23:28,554 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T12:23:28,554 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T12:23:28,554 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,554 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:28,554 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,554 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,554 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,554 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:28,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:28,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/eeb4f1bd0ac848bba4b1033041220fa5 2024-11-07T12:23:28,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/73eeed12af8c434aacb55972ea552439 is 50, key is test_row_0/B:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:28,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741925_1101 (size=12301) 2024-11-07T12:23:29,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/73eeed12af8c434aacb55972ea552439 2024-11-07T12:23:29,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/67a87d3397bd4f848f02d960e01ba98a is 50, key is test_row_0/C:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:29,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741926_1102 (size=12301) 2024-11-07T12:23:29,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/67a87d3397bd4f848f02d960e01ba98a 2024-11-07T12:23:29,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/eeb4f1bd0ac848bba4b1033041220fa5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5 2024-11-07T12:23:29,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5, entries=150, sequenceid=429, filesize=12.0 K 2024-11-07T12:23:29,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/73eeed12af8c434aacb55972ea552439 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439 2024-11-07T12:23:29,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439, entries=150, sequenceid=429, filesize=12.0 K 2024-11-07T12:23:29,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/67a87d3397bd4f848f02d960e01ba98a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a 2024-11-07T12:23:29,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a, entries=150, sequenceid=429, filesize=12.0 K 2024-11-07T12:23:29,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=13.42 KB/13740 for 533db45226dd865e0c0b2154b214ffef in 1272ms, sequenceid=429, compaction requested=true 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:29,695 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 533db45226dd865e0c0b2154b214ffef:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:29,695 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/A is initiating minor compaction (all files) 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/B is initiating minor compaction (all files) 2024-11-07T12:23:29,696 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/A in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:29,696 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/B in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:29,696 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/158568a93236439eb45efbc6a8da6e7d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=36.9 K 2024-11-07T12:23:29,696 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c58cf1fc08de408ab4b3878ec0be4a34, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=36.9 K 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158568a93236439eb45efbc6a8da6e7d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:29,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c58cf1fc08de408ab4b3878ec0be4a34, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:29,697 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318d5cf8e20d47388cce5cb38a238d01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1730982206246 2024-11-07T12:23:29,697 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 098a836e1ca14678af9826bd9720f770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1730982206246 2024-11-07T12:23:29,697 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb4f1bd0ac848bba4b1033041220fa5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1730982207412 2024-11-07T12:23:29,697 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 73eeed12af8c434aacb55972ea552439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1730982207412 2024-11-07T12:23:29,706 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:29,706 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:29,706 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2d2243e2b1ab4cda9d9b137b2df56789 is 50, key is test_row_0/A:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:29,706 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/dd349ac76dce487db03b4893edd16dde is 50, key is test_row_0/B:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741927_1103 (size=13323) 2024-11-07T12:23:29,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741928_1104 (size=13323) 2024-11-07T12:23:30,116 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/dd349ac76dce487db03b4893edd16dde as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/dd349ac76dce487db03b4893edd16dde 2024-11-07T12:23:30,119 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/2d2243e2b1ab4cda9d9b137b2df56789 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2d2243e2b1ab4cda9d9b137b2df56789 2024-11-07T12:23:30,122 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/B of 533db45226dd865e0c0b2154b214ffef into dd349ac76dce487db03b4893edd16dde(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:30,122 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/B, priority=13, startTime=1730982209695; duration=0sec 2024-11-07T12:23:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:B 2024-11-07T12:23:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 533db45226dd865e0c0b2154b214ffef/C is initiating minor compaction (all files) 2024-11-07T12:23:30,123 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 533db45226dd865e0c0b2154b214ffef/C in TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:30,123 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/101c2174e75d4dcd8a6d51f0a5084370, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp, totalSize=36.9 K 2024-11-07T12:23:30,124 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 101c2174e75d4dcd8a6d51f0a5084370, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730982205589 2024-11-07T12:23:30,125 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b92ada5670c74e38883fa9e005b51f9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1730982206246 2024-11-07T12:23:30,125 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/A of 533db45226dd865e0c0b2154b214ffef into 2d2243e2b1ab4cda9d9b137b2df56789(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:30,125 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:30,125 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a87d3397bd4f848f02d960e01ba98a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1730982207412 2024-11-07T12:23:30,125 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/A, priority=13, startTime=1730982209695; duration=0sec 2024-11-07T12:23:30,125 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:30,125 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:A 2024-11-07T12:23:30,132 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 533db45226dd865e0c0b2154b214ffef#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:30,132 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/58f6cf0e927342d28d7e47d73bf13281 is 50, key is test_row_0/C:col10/1730982208421/Put/seqid=0 2024-11-07T12:23:30,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741929_1105 (size=13323) 2024-11-07T12:23:30,490 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:23:30,542 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/58f6cf0e927342d28d7e47d73bf13281 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/58f6cf0e927342d28d7e47d73bf13281 2024-11-07T12:23:30,547 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 533db45226dd865e0c0b2154b214ffef/C of 533db45226dd865e0c0b2154b214ffef into 58f6cf0e927342d28d7e47d73bf13281(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:30,547 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 533db45226dd865e0c0b2154b214ffef: 2024-11-07T12:23:30,547 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., storeName=533db45226dd865e0c0b2154b214ffef/C, priority=13, startTime=1730982209695; duration=0sec 2024-11-07T12:23:30,547 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:30,547 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 533db45226dd865e0c0b2154b214ffef:C 2024-11-07T12:23:37,082 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T12:23:37,083 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:23:37,083 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:37,083 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:37,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3883 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3870 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:23:37,083 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1742 2024-11-07T12:23:37,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5216 rows 2024-11-07T12:23:37,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1748 2024-11-07T12:23:37,084 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5237 rows 2024-11-07T12:23:37,084 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:23:37,084 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:23:37,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:37,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:37,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:23:37,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:23:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:37,093 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982217093"}]},"ts":"1730982217093"} 2024-11-07T12:23:37,095 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:23:37,095 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:23:37,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:23:37,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, UNASSIGN}] 2024-11-07T12:23:37,102 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, UNASSIGN 2024-11-07T12:23:37,103 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=533db45226dd865e0c0b2154b214ffef, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:37,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, UNASSIGN because future has completed 2024-11-07T12:23:37,106 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:23:37,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:37,262 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:37,262 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:23:37,263 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 533db45226dd865e0c0b2154b214ffef, disabling compactions & flushes 2024-11-07T12:23:37,263 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:37,263 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:37,263 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. after waiting 0 ms 2024-11-07T12:23:37,263 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:37,263 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 533db45226dd865e0c0b2154b214ffef 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T12:23:37,267 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e64a2828c90342788e830eff037905b1 is 50, key is test_row_0/A:col10/1730982208551/Put/seqid=0 2024-11-07T12:23:37,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741930_1106 (size=12301) 2024-11-07T12:23:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:37,672 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e64a2828c90342788e830eff037905b1 2024-11-07T12:23:37,681 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/66542214a80a41659fb7ffee956d1231 is 50, key is test_row_0/B:col10/1730982208551/Put/seqid=0 2024-11-07T12:23:37,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741931_1107 (size=12301) 2024-11-07T12:23:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:38,086 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/66542214a80a41659fb7ffee956d1231 2024-11-07T12:23:38,093 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/483f5802dfce4b4a8541ea6401206773 is 50, key is test_row_0/C:col10/1730982208551/Put/seqid=0 2024-11-07T12:23:38,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741932_1108 (size=12301) 2024-11-07T12:23:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:38,498 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/483f5802dfce4b4a8541ea6401206773 2024-11-07T12:23:38,503 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/A/e64a2828c90342788e830eff037905b1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e64a2828c90342788e830eff037905b1 2024-11-07T12:23:38,507 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e64a2828c90342788e830eff037905b1, entries=150, sequenceid=438, filesize=12.0 K 2024-11-07T12:23:38,508 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/B/66542214a80a41659fb7ffee956d1231 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/66542214a80a41659fb7ffee956d1231 2024-11-07T12:23:38,513 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/66542214a80a41659fb7ffee956d1231, entries=150, sequenceid=438, filesize=12.0 K 2024-11-07T12:23:38,514 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/.tmp/C/483f5802dfce4b4a8541ea6401206773 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/483f5802dfce4b4a8541ea6401206773 2024-11-07T12:23:38,520 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/483f5802dfce4b4a8541ea6401206773, entries=150, sequenceid=438, filesize=12.0 K 2024-11-07T12:23:38,521 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 533db45226dd865e0c0b2154b214ffef in 1258ms, sequenceid=438, compaction requested=false 2024-11-07T12:23:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c8ee942a8b704652bf344d354c6c597d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2abb8ee9fa5444e4baa830edaecf4d19, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d30d89a8ff874df886f0e1533d0b9c3a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0691f6754bab4c27aa63bb0df0d52197, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/70fcb64f5844443db0d5f0b6b7779bb3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/129550d89d574457930e47c0316226f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/ff5ba0ca58414356822366408625f0e6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/158568a93236439eb45efbc6a8da6e7d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5] to archive 2024-11-07T12:23:38,525 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:23:38,530 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/86a0c8b449b64cbfb82ce32cfe29af3d 2024-11-07T12:23:38,531 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e775fcc57a544038a311e735452cae31 2024-11-07T12:23:38,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c8ee942a8b704652bf344d354c6c597d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c8ee942a8b704652bf344d354c6c597d 2024-11-07T12:23:38,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16e8525782364b31b6dad3b0ae3ba3b6 2024-11-07T12:23:38,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/094d5550576f4332bee8daf62bc24e13 2024-11-07T12:23:38,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2abb8ee9fa5444e4baa830edaecf4d19 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2abb8ee9fa5444e4baa830edaecf4d19 2024-11-07T12:23:38,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d3512ca98e1345da891414e049b96633 2024-11-07T12:23:38,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/93b02a01018d44e6831361d87e4448a2 2024-11-07T12:23:38,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/7f938ef91f7241c6af772732140dc09e 2024-11-07T12:23:38,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/16a543e45f364462a7d6dfa8e12c4dc1 2024-11-07T12:23:38,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d30d89a8ff874df886f0e1533d0b9c3a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d30d89a8ff874df886f0e1533d0b9c3a 2024-11-07T12:23:38,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/1118870ef95e47a1849c4a5b894e901c 2024-11-07T12:23:38,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0691f6754bab4c27aa63bb0df0d52197 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0691f6754bab4c27aa63bb0df0d52197 2024-11-07T12:23:38,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/065435cdd5d2454d8cc65da37d7c9ccb 2024-11-07T12:23:38,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/c537bccd6a694638ad3bd56b12c8bafb 2024-11-07T12:23:38,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/915ba3472dfd4e71bd647b42ca05960c 2024-11-07T12:23:38,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d00c8b8069374e2fb1becbaff0b7718e 2024-11-07T12:23:38,551 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/70fcb64f5844443db0d5f0b6b7779bb3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/70fcb64f5844443db0d5f0b6b7779bb3 2024-11-07T12:23:38,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2fd929dd586b49d2bcb9b70adfc4f625 2024-11-07T12:23:38,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/129550d89d574457930e47c0316226f4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/129550d89d574457930e47c0316226f4 2024-11-07T12:23:38,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/6d282149189a43ae956cdef8f1531862 2024-11-07T12:23:38,554 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/d223705a4c1a459587be1581861d0409 2024-11-07T12:23:38,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/5bff6723e35e4f5da035e2b884441d0e 2024-11-07T12:23:38,556 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/796e7da57fce4a8f835faa6bfb490592 2024-11-07T12:23:38,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/ff5ba0ca58414356822366408625f0e6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/ff5ba0ca58414356822366408625f0e6 2024-11-07T12:23:38,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2f5410d595974e558a199b28509c7338 2024-11-07T12:23:38,560 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/0ef22f9527644a51adb8d692fd2837e4 2024-11-07T12:23:38,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/158568a93236439eb45efbc6a8da6e7d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/158568a93236439eb45efbc6a8da6e7d 2024-11-07T12:23:38,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/318d5cf8e20d47388cce5cb38a238d01 2024-11-07T12:23:38,563 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/eeb4f1bd0ac848bba4b1033041220fa5 2024-11-07T12:23:38,579 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3bf6dd289fba46fabdbf55469b8bc32d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a74a19f9a4364682a34209ad6f5158b1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/638bee12faaa4706b82171aa18a66044, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6794dfcd78234b129c8f91a9f697d147, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/4100b5cb597448819541659716ba0c41, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/891109e808e848dcac5e6d5aed56eab3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9b185515239d4e3aa6b96238eabaf17a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c58cf1fc08de408ab4b3878ec0be4a34, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439] to archive 2024-11-07T12:23:38,580 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:23:38,582 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2117d0c472d947dc9e25c92965333f64 2024-11-07T12:23:38,583 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/34be93cc666d47f7844a4618ce90b43a 2024-11-07T12:23:38,585 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3bf6dd289fba46fabdbf55469b8bc32d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3bf6dd289fba46fabdbf55469b8bc32d 2024-11-07T12:23:38,586 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/13041a1baca449909a8711cd7e413e0a 2024-11-07T12:23:38,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9c059a3e77214993aca502a31314563f 2024-11-07T12:23:38,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a74a19f9a4364682a34209ad6f5158b1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a74a19f9a4364682a34209ad6f5158b1 2024-11-07T12:23:38,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/37ff839b67bf4a34abf08d9cf8153b4e 2024-11-07T12:23:38,591 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/a9484b9073c541aca8711f16daa25820 2024-11-07T12:23:38,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/787e762b9d5348a383fbdc7bcd913dcf 2024-11-07T12:23:38,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/638bee12faaa4706b82171aa18a66044 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/638bee12faaa4706b82171aa18a66044 2024-11-07T12:23:38,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/598903122f0b4f31928938fc3f844c79 2024-11-07T12:23:38,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c4d01a7603214506b9685808db3a37f7 2024-11-07T12:23:38,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6794dfcd78234b129c8f91a9f697d147 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6794dfcd78234b129c8f91a9f697d147 2024-11-07T12:23:38,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/acbb5ab726a84732b85271a99389488e 2024-11-07T12:23:38,600 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/399be3c3e74e4cf5b34b990afd31bd5b 2024-11-07T12:23:38,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/2ff96b42dd4c4c09a0b8a81dced2c78a 2024-11-07T12:23:38,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/4100b5cb597448819541659716ba0c41 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/4100b5cb597448819541659716ba0c41 2024-11-07T12:23:38,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/62b9049cb06e4ef7ad28f803287530ad 2024-11-07T12:23:38,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/e42a28bee625434bb76467ed33e773d1 2024-11-07T12:23:38,606 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/891109e808e848dcac5e6d5aed56eab3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/891109e808e848dcac5e6d5aed56eab3 2024-11-07T12:23:38,608 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/00f3a8aa93c04a83a715225aec6ce58b 2024-11-07T12:23:38,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/583204f998a149db9151c45f646e1443 2024-11-07T12:23:38,610 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/3f26426a750b4a8b951f7ad15e162ca4 2024-11-07T12:23:38,612 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9b185515239d4e3aa6b96238eabaf17a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/9b185515239d4e3aa6b96238eabaf17a 2024-11-07T12:23:38,613 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/951757aaced14211ab0c5c6b1494923f 2024-11-07T12:23:38,615 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/72bab7fbeb40425aa5fa857b6f843c53 2024-11-07T12:23:38,616 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c58cf1fc08de408ab4b3878ec0be4a34 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/c58cf1fc08de408ab4b3878ec0be4a34 2024-11-07T12:23:38,618 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/6b15e69b39994e64b93d7cf0ab7c7633 2024-11-07T12:23:38,619 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/098a836e1ca14678af9826bd9720f770 2024-11-07T12:23:38,620 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/73eeed12af8c434aacb55972ea552439 2024-11-07T12:23:38,622 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/8b4d9792bc684efb91ae2747e1e2b25c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3b3d5833888141bbb10e79c7562b02d4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c857da2e4cd64977a34a329a836563c9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3f6fa00bb6644693b101788283fe761e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68ce989ac7384f308fd9a91c3a40344f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/db8e5feaaaaa4038a31c6b0add349e9e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7edbb641e3d34240a35385db9e8eb996, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/101c2174e75d4dcd8a6d51f0a5084370, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a] to archive 2024-11-07T12:23:38,623 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:23:38,625 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e8df4381fca94c799b633e7b7083d13e 2024-11-07T12:23:38,627 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/f77274fb4eb24dc995fc73931fffe062 2024-11-07T12:23:38,628 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/8b4d9792bc684efb91ae2747e1e2b25c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/8b4d9792bc684efb91ae2747e1e2b25c 2024-11-07T12:23:38,629 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e01906c429b94d1782cbb89d9fac66ea 2024-11-07T12:23:38,630 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b0400d4008824cf28b723a6f9d0eb6b3 2024-11-07T12:23:38,632 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3b3d5833888141bbb10e79c7562b02d4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3b3d5833888141bbb10e79c7562b02d4 2024-11-07T12:23:38,633 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a7332e797e654da1a3ed9d773259b5b1 2024-11-07T12:23:38,634 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/a740119d818a495f852ee81ca651f80e 2024-11-07T12:23:38,635 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/242d1f5dcd0745c080881f647761dca5 2024-11-07T12:23:38,636 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c857da2e4cd64977a34a329a836563c9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c857da2e4cd64977a34a329a836563c9 2024-11-07T12:23:38,638 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/17cc45c6363d45d092cffa1196b75ab1 2024-11-07T12:23:38,639 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/e6383d77c8b84094ac0b0272436642aa 2024-11-07T12:23:38,640 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3f6fa00bb6644693b101788283fe761e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/3f6fa00bb6644693b101788283fe761e 2024-11-07T12:23:38,641 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4fc023e8a86c41148c984cc37e006514 2024-11-07T12:23:38,642 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/5e94a36a5c084268a7979088ed21af7b 2024-11-07T12:23:38,644 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/4125660586254009868b71bfbd098546 2024-11-07T12:23:38,645 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68ce989ac7384f308fd9a91c3a40344f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68ce989ac7384f308fd9a91c3a40344f 2024-11-07T12:23:38,646 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/22b25aac8e724ea98e1afa8091ba214a 2024-11-07T12:23:38,648 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/c458bb66b0ae41db82f79e1c95890c55 2024-11-07T12:23:38,649 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/db8e5feaaaaa4038a31c6b0add349e9e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/db8e5feaaaaa4038a31c6b0add349e9e 2024-11-07T12:23:38,650 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7b476ebd970f4e468208420ca5446cce 2024-11-07T12:23:38,651 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/650e5b9b07d44caeb523e92114a619bf 2024-11-07T12:23:38,652 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/598697cb069146baaaa6e72861be9d18 2024-11-07T12:23:38,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7edbb641e3d34240a35385db9e8eb996 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/7edbb641e3d34240a35385db9e8eb996 2024-11-07T12:23:38,654 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/26cc8d24361d4bcebb45d69a185350f1 2024-11-07T12:23:38,655 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/68bb9c62de494a09b1539a06cc7cbc06 2024-11-07T12:23:38,657 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/101c2174e75d4dcd8a6d51f0a5084370 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/101c2174e75d4dcd8a6d51f0a5084370 2024-11-07T12:23:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/bb7ba570be33423a83641ddcc31f8b77 2024-11-07T12:23:38,659 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/b92ada5670c74e38883fa9e005b51f9c 2024-11-07T12:23:38,661 DEBUG [StoreCloser-TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/67a87d3397bd4f848f02d960e01ba98a 2024-11-07T12:23:38,666 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/recovered.edits/441.seqid, newMaxSeqId=441, maxSeqId=1 2024-11-07T12:23:38,667 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. 2024-11-07T12:23:38,667 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 533db45226dd865e0c0b2154b214ffef: Waiting for close lock at 1730982217262Running coprocessor pre-close hooks at 1730982217262Disabling compacts and flushes for region at 1730982217263 (+1 ms)Disabling writes for close at 1730982217263Obtaining lock to block concurrent updates at 1730982217263Preparing flush snapshotting stores in 533db45226dd865e0c0b2154b214ffef at 1730982217263Finished memstore snapshotting TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730982217263Flushing stores of TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef. at 1730982217264 (+1 ms)Flushing 533db45226dd865e0c0b2154b214ffef/A: creating writer at 1730982217264Flushing 533db45226dd865e0c0b2154b214ffef/A: appending metadata at 1730982217267 (+3 ms)Flushing 533db45226dd865e0c0b2154b214ffef/A: closing flushed file at 1730982217267Flushing 533db45226dd865e0c0b2154b214ffef/B: creating writer at 1730982217677 (+410 ms)Flushing 533db45226dd865e0c0b2154b214ffef/B: appending metadata at 1730982217680 (+3 ms)Flushing 533db45226dd865e0c0b2154b214ffef/B: closing flushed file at 1730982217680Flushing 533db45226dd865e0c0b2154b214ffef/C: creating writer at 1730982218090 (+410 ms)Flushing 533db45226dd865e0c0b2154b214ffef/C: appending metadata at 1730982218093 (+3 ms)Flushing 533db45226dd865e0c0b2154b214ffef/C: closing flushed file at 1730982218093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1db601a0: reopening flushed file at 1730982218502 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3969d6b5: reopening flushed file at 1730982218507 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63530578: reopening flushed file at 1730982218513 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 533db45226dd865e0c0b2154b214ffef in 1258ms, sequenceid=438, compaction requested=false at 1730982218521 (+8 ms)Writing region close event to WAL at 1730982218662 (+141 ms)Running coprocessor post-close hooks at 1730982218666 (+4 ms)Closed at 1730982218666 2024-11-07T12:23:38,669 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:38,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=533db45226dd865e0c0b2154b214ffef, regionState=CLOSED 2024-11-07T12:23:38,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:23:38,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T12:23:38,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 533db45226dd865e0c0b2154b214ffef, server=8944cbb06cd2,46221,1730982182218 in 1.5670 sec 2024-11-07T12:23:38,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T12:23:38,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=533db45226dd865e0c0b2154b214ffef, UNASSIGN in 1.5740 sec 2024-11-07T12:23:38,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T12:23:38,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5800 sec 2024-11-07T12:23:38,680 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982218679"}]},"ts":"1730982218679"} 2024-11-07T12:23:38,681 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:23:38,681 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:23:38,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5940 sec 2024-11-07T12:23:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T12:23:39,252 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:39,259 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:23:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,267 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T12:23:39,269 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,274 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:39,279 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/recovered.edits] 2024-11-07T12:23:39,282 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2d2243e2b1ab4cda9d9b137b2df56789 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/2d2243e2b1ab4cda9d9b137b2df56789 2024-11-07T12:23:39,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e64a2828c90342788e830eff037905b1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/A/e64a2828c90342788e830eff037905b1 2024-11-07T12:23:39,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/66542214a80a41659fb7ffee956d1231 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/66542214a80a41659fb7ffee956d1231 2024-11-07T12:23:39,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/dd349ac76dce487db03b4893edd16dde to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/B/dd349ac76dce487db03b4893edd16dde 2024-11-07T12:23:39,290 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/483f5802dfce4b4a8541ea6401206773 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/483f5802dfce4b4a8541ea6401206773 2024-11-07T12:23:39,291 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/58f6cf0e927342d28d7e47d73bf13281 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/C/58f6cf0e927342d28d7e47d73bf13281 2024-11-07T12:23:39,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/recovered.edits/441.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef/recovered.edits/441.seqid 2024-11-07T12:23:39,295 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/533db45226dd865e0c0b2154b214ffef 2024-11-07T12:23:39,295 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:23:39,300 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T12:23:39,307 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:23:39,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:23:39,333 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,334 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:23:39,334 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982219334"}]},"ts":"9223372036854775807"} 2024-11-07T12:23:39,337 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:23:39,337 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 533db45226dd865e0c0b2154b214ffef, NAME => 'TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:23:39,338 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:23:39,338 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982219338"}]},"ts":"9223372036854775807"} 2024-11-07T12:23:39,341 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:23:39,343 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-07T12:23:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T12:23:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:23:39,372 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:39,382 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1393209540_22 at /127.0.0.1:47672 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8944cbb06cd2:46221-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=605 (was 769), ProcessCount=11 (was 11), AvailableMemoryMB=1670 (was 2250) 2024-11-07T12:23:39,391 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=605, ProcessCount=11, AvailableMemoryMB=1669 2024-11-07T12:23:39,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:23:39,393 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:23:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:39,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:23:39,395 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:39,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-07T12:23:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:39,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:23:39,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741933_1109 (size=959) 2024-11-07T12:23:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:39,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:23:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741934_1110 (size=53) 2024-11-07T12:23:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:40,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:40,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 548e342ecfd588a5f02a3dbe0c2bf2ae, disabling compactions & flushes 2024-11-07T12:23:40,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. after waiting 0 ms 2024-11-07T12:23:40,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: Waiting for close lock at 1730982220213Disabling compacts and flushes for region at 1730982220213Disabling writes for close at 1730982220214 (+1 ms)Writing region close event to WAL at 1730982220214Closed at 1730982220214 2024-11-07T12:23:40,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:23:40,216 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982220215"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982220215"}]},"ts":"1730982220215"} 2024-11-07T12:23:40,218 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:23:40,220 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:23:40,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982220220"}]},"ts":"1730982220220"} 2024-11-07T12:23:40,223 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:23:40,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, ASSIGN}] 2024-11-07T12:23:40,225 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, ASSIGN 2024-11-07T12:23:40,226 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:23:40,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:40,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, ASSIGN because future has completed 2024-11-07T12:23:40,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:40,536 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,536 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:23:40,537 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,537 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:40,537 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,537 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,539 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,542 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName A 2024-11-07T12:23:40,543 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:40,544 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:40,544 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,546 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName B 2024-11-07T12:23:40,546 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:40,546 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:40,547 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,548 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName C 2024-11-07T12:23:40,548 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:40,549 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:40,549 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,550 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,550 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,552 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,552 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,552 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:23:40,554 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,557 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:23:40,557 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 548e342ecfd588a5f02a3dbe0c2bf2ae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61783869, jitterRate=-0.0793486088514328}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:23:40,557 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:40,559 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: Running coprocessor pre-open hook at 1730982220537Writing region info on filesystem at 1730982220537Initializing all the Stores at 1730982220539 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982220539Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982220539Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982220539Cleaning up temporary data from old regions at 1730982220552 (+13 ms)Running coprocessor post-open hooks at 1730982220557 (+5 ms)Region opened successfully at 1730982220558 (+1 ms) 2024-11-07T12:23:40,559 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., pid=34, masterSystemTime=1730982220531 2024-11-07T12:23:40,562 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,562 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:40,563 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:40,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:23:40,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T12:23:40,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 in 187 msec 2024-11-07T12:23:40,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-07T12:23:40,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, ASSIGN in 345 msec 2024-11-07T12:23:40,572 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:23:40,572 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982220572"}]},"ts":"1730982220572"} 2024-11-07T12:23:40,574 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:23:40,575 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:23:40,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-07T12:23:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T12:23:41,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:41,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e732f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:41,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:41,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:41,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e67fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:41,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:41,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dc58f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:41,550 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:41,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:41,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:23:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:23:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51700bd0 2024-11-07T12:23:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:23:41,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:23:41,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:23:41,565 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:23:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:23:41,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741935_1111 (size=995) 2024-11-07T12:23:41,983 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:23:41,983 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:23:41,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:23:41,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE}] 2024-11-07T12:23:41,996 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE 2024-11-07T12:23:41,997 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:41,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE because future has completed 2024-11-07T12:23:41,999 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:23:41,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:42,152 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,152 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:23:42,152 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 548e342ecfd588a5f02a3dbe0c2bf2ae, disabling compactions & flushes 2024-11-07T12:23:42,152 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,153 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,153 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. after waiting 0 ms 2024-11-07T12:23:42,153 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,157 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:23:42,157 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,157 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: Waiting for close lock at 1730982222152Running coprocessor pre-close hooks at 1730982222152Disabling compacts and flushes for region at 1730982222152Disabling writes for close at 1730982222153 (+1 ms)Writing region close event to WAL at 1730982222153Running coprocessor post-close hooks at 1730982222157 (+4 ms)Closed at 1730982222157 2024-11-07T12:23:42,158 WARN [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 548e342ecfd588a5f02a3dbe0c2bf2ae to self. 2024-11-07T12:23:42,159 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=CLOSED 2024-11-07T12:23:42,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:23:42,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T12:23:42,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 in 163 msec 2024-11-07T12:23:42,165 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE; state=CLOSED, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=true 2024-11-07T12:23:42,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE because future has completed 2024-11-07T12:23:42,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:23:42,472 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,472 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:23:42,473 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,473 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:23:42,473 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,473 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,477 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,481 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName A 2024-11-07T12:23:42,483 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:42,483 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:42,484 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,484 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName B 2024-11-07T12:23:42,485 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:42,485 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:42,485 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,486 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 548e342ecfd588a5f02a3dbe0c2bf2ae columnFamilyName C 2024-11-07T12:23:42,486 DEBUG [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:42,486 INFO [StoreOpener-548e342ecfd588a5f02a3dbe0c2bf2ae-1 {}] regionserver.HStore(327): Store=548e342ecfd588a5f02a3dbe0c2bf2ae/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:23:42,486 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,487 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,488 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,489 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,489 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,489 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:23:42,490 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,491 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 548e342ecfd588a5f02a3dbe0c2bf2ae; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63112419, jitterRate=-0.0595516711473465}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:23:42,491 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,492 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: Running coprocessor pre-open hook at 1730982222473Writing region info on filesystem at 1730982222473Initializing all the Stores at 1730982222474 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982222474Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982222474Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982222474Cleaning up temporary data from old regions at 1730982222489 (+15 ms)Running coprocessor post-open hooks at 1730982222491 (+2 ms)Region opened successfully at 1730982222492 (+1 ms) 2024-11-07T12:23:42,493 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., pid=39, masterSystemTime=1730982222469 2024-11-07T12:23:42,494 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,494 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=OPEN, openSeqNum=5, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:23:42,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-07T12:23:42,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 in 180 msec 2024-11-07T12:23:42,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T12:23:42,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, REOPEN/MOVE in 504 msec 2024-11-07T12:23:42,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-07T12:23:42,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-07T12:23:42,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-11-07T12:23:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-07T12:23:42,509 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:42,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4716031f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,510 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ababc3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118c8dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d3ba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7a241d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,516 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2458748b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ccd683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40553111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,520 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1992715b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47250f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,522 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c719083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c023a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,525 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f21568d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,527 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4aff27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed913a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,529 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7f49fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33430329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49519de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,532 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfc90db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,533 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6180c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16801009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,535 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13eaea15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee88c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7183633a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:23:42,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:23:42,539 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbd1182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:23:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:23:42,540 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:23:42,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38236e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:23:42,541 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,541 DEBUG [Thread-583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,541 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,542 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,542 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,542 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,544 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,545 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:42,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,547 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-07T12:23:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,548 DEBUG [Thread-575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:23:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:42,549 DEBUG [Thread-575 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:23:42,550 DEBUG [Thread-575 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:23:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,550 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:42,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:42,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:23:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:23:42,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:42,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774b534dbb6804b72a563c7b63a9cb496_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982222570/Put/seqid=0 2024-11-07T12:23:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982282631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982282633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982282634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982282636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982282645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:42,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741936_1112 (size=14594) 2024-11-07T12:23:42,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:42,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774b534dbb6804b72a563c7b63a9cb496_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b534dbb6804b72a563c7b63a9cb496_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:42,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/cb6cb436a0cd49538e865fe09839be91, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:42,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/cb6cb436a0cd49538e865fe09839be91 is 175, key is test_row_0/A:col10/1730982222570/Put/seqid=0 2024-11-07T12:23:42,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741937_1113 (size=39549) 2024-11-07T12:23:42,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982282751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982282762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982282762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982282762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982282762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:42,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982282971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982282971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982282981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982282982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982282982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,134 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/cb6cb436a0cd49538e865fe09839be91 2024-11-07T12:23:43,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/0aadec8cb9994c128c5d022414b7ddbc is 50, key is test_row_0/B:col10/1730982222570/Put/seqid=0 2024-11-07T12:23:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:43,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741938_1114 (size=12001) 2024-11-07T12:23:43,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/0aadec8cb9994c128c5d022414b7ddbc 2024-11-07T12:23:43,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/a4c831cb789549f1a879df01bc1e6719 is 50, key is test_row_0/C:col10/1730982222570/Put/seqid=0 2024-11-07T12:23:43,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741939_1115 (size=12001) 2024-11-07T12:23:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982283291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982283301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:43,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/a4c831cb789549f1a879df01bc1e6719 2024-11-07T12:23:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:43,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/cb6cb436a0cd49538e865fe09839be91 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91 2024-11-07T12:23:43,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91, entries=200, sequenceid=16, filesize=38.6 K 2024-11-07T12:23:43,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/0aadec8cb9994c128c5d022414b7ddbc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc 2024-11-07T12:23:43,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:23:43,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/a4c831cb789549f1a879df01bc1e6719 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719 2024-11-07T12:23:43,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T12:23:43,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1181ms, sequenceid=16, compaction requested=false 2024-11-07T12:23:43,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:43,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T12:23:43,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:43,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:23:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:43,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:43,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b238e2803aa24fa2917c72b56caa38d5_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982222632/Put/seqid=0 2024-11-07T12:23:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982283824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982283826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982283830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982283836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982283835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741940_1116 (size=12154) 2024-11-07T12:23:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982283940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982283941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982283941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982283951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982283952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:43,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982284151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982284151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982284151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982284160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982284160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,235 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:23:44,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:44,290 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b238e2803aa24fa2917c72b56caa38d5_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b238e2803aa24fa2917c72b56caa38d5_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:44,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/3a93f6ffe9ab4e798763488faa2a4939, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/3a93f6ffe9ab4e798763488faa2a4939 is 175, key is test_row_0/A:col10/1730982222632/Put/seqid=0 2024-11-07T12:23:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741941_1117 (size=30955) 2024-11-07T12:23:44,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/3a93f6ffe9ab4e798763488faa2a4939 2024-11-07T12:23:44,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/535f327f6491422c8d194f9f9f864986 is 50, key is test_row_0/B:col10/1730982222632/Put/seqid=0 2024-11-07T12:23:44,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741942_1118 (size=12001) 2024-11-07T12:23:44,324 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/535f327f6491422c8d194f9f9f864986 2024-11-07T12:23:44,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/baa5fb0fcb6644e09e2787ca0fc1ebbf is 50, key is test_row_0/C:col10/1730982222632/Put/seqid=0 2024-11-07T12:23:44,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741943_1119 (size=12001) 2024-11-07T12:23:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982284461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982284461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982284461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982284471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982284471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:44,764 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/baa5fb0fcb6644e09e2787ca0fc1ebbf 2024-11-07T12:23:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/3a93f6ffe9ab4e798763488faa2a4939 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939 2024-11-07T12:23:44,779 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T12:23:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/535f327f6491422c8d194f9f9f864986 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986 2024-11-07T12:23:44,806 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:23:44,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/baa5fb0fcb6644e09e2787ca0fc1ebbf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf 2024-11-07T12:23:44,816 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:23:44,818 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1026ms, sequenceid=40, compaction requested=false 2024-11-07T12:23:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:44,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-07T12:23:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-07T12:23:44,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T12:23:44,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2690 sec 2024-11-07T12:23:44,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.2810 sec 2024-11-07T12:23:44,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:44,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:44,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:23:44,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d3e86f1c4ec14148bba0a5de17531840_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982285006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982285006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982285007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741944_1120 (size=14594) 2024-11-07T12:23:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982285011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982285014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:45,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d3e86f1c4ec14148bba0a5de17531840_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3e86f1c4ec14148bba0a5de17531840_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:45,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0ab99efd8fe94bd9bfaf7360798f7587, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0ab99efd8fe94bd9bfaf7360798f7587 is 175, key is test_row_0/A:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741945_1121 (size=39549) 2024-11-07T12:23:45,046 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0ab99efd8fe94bd9bfaf7360798f7587 2024-11-07T12:23:45,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/25ef12b9dee440b29b9ec1a9baa6c5fd is 50, key is test_row_0/B:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741946_1122 (size=12001) 2024-11-07T12:23:45,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/25ef12b9dee440b29b9ec1a9baa6c5fd 2024-11-07T12:23:45,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f9e566636b3d40cc80c3f426def9ca88 is 50, key is test_row_0/C:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741947_1123 (size=12001) 2024-11-07T12:23:45,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982285120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982285120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982285121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982285131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982285131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982285331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982285331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982285332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982285341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982285341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f9e566636b3d40cc80c3f426def9ca88 2024-11-07T12:23:45,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0ab99efd8fe94bd9bfaf7360798f7587 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587 2024-11-07T12:23:45,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587, entries=200, sequenceid=53, filesize=38.6 K 2024-11-07T12:23:45,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/25ef12b9dee440b29b9ec1a9baa6c5fd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd 2024-11-07T12:23:45,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T12:23:45,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f9e566636b3d40cc80c3f426def9ca88 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88 2024-11-07T12:23:45,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T12:23:45,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 566ms, sequenceid=53, compaction requested=true 2024-11-07T12:23:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:45,540 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:45,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:45,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:45,541 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:45,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:45,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:45,543 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:45,543 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:45,543 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:45,543 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:45,543 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:45,543 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:45,543 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.2 K 2024-11-07T12:23:45,543 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=107.5 K 2024-11-07T12:23:45,543 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:45,544 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587] 2024-11-07T12:23:45,544 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aadec8cb9994c128c5d022414b7ddbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982222568 2024-11-07T12:23:45,544 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6cb436a0cd49538e865fe09839be91, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982222566 2024-11-07T12:23:45,544 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a93f6ffe9ab4e798763488faa2a4939, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982222632 2024-11-07T12:23:45,545 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 535f327f6491422c8d194f9f9f864986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982222632 2024-11-07T12:23:45,545 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab99efd8fe94bd9bfaf7360798f7587, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223821 2024-11-07T12:23:45,545 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ef12b9dee440b29b9ec1a9baa6c5fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223828 2024-11-07T12:23:45,561 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:45,562 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/a57dd3586ae844ff90e969132d3327f7 is 50, key is test_row_0/B:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,564 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,575 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411072ecf4cb6da3044cfa925fe9486e375ec_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741948_1124 (size=12104) 2024-11-07T12:23:45,585 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411072ecf4cb6da3044cfa925fe9486e375ec_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,586 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ecf4cb6da3044cfa925fe9486e375ec_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741949_1125 (size=4469) 2024-11-07T12:23:45,637 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#106 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:45,638 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fe9e587dc14a4662a318618add96c721 is 175, key is test_row_0/A:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:45,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:45,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982285658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982285658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982285660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982285663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982285668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741950_1126 (size=31058) 2024-11-07T12:23:45,690 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fe9e587dc14a4662a318618add96c721 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721 2024-11-07T12:23:45,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077fa4b157addd43a38fdb772ec2ca11dd_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982225009/Put/seqid=0 2024-11-07T12:23:45,704 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into fe9e587dc14a4662a318618add96c721(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:45,704 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:45,705 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982225540; duration=0sec 2024-11-07T12:23:45,705 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:45,705 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:45,705 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:45,706 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:45,707 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:45,707 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:45,707 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.2 K 2024-11-07T12:23:45,707 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c831cb789549f1a879df01bc1e6719, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730982222568 2024-11-07T12:23:45,707 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa5fb0fcb6644e09e2787ca0fc1ebbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982222632 2024-11-07T12:23:45,708 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e566636b3d40cc80c3f426def9ca88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223828 2024-11-07T12:23:45,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741951_1127 (size=17034) 2024-11-07T12:23:45,738 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:45,738 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:45,739 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/0de46198431b451582c954dcd14b901b is 50, key is test_row_0/C:col10/1730982223830/Put/seqid=0 2024-11-07T12:23:45,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077fa4b157addd43a38fdb772ec2ca11dd_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077fa4b157addd43a38fdb772ec2ca11dd_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:45,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dee275d585cd4734ae3d4ea3e0d5055a, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:45,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dee275d585cd4734ae3d4ea3e0d5055a is 175, key is test_row_0/A:col10/1730982225009/Put/seqid=0 2024-11-07T12:23:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982285771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982285771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982285781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741952_1128 (size=12104) 2024-11-07T12:23:45,806 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/0de46198431b451582c954dcd14b901b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0de46198431b451582c954dcd14b901b 2024-11-07T12:23:45,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741953_1129 (size=48139) 2024-11-07T12:23:45,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dee275d585cd4734ae3d4ea3e0d5055a 2024-11-07T12:23:45,815 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 0de46198431b451582c954dcd14b901b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:45,815 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:45,815 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982225541; duration=0sec 2024-11-07T12:23:45,815 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:45,816 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:45,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e9f1cbb032064b00a069c429d14ac1ec is 50, key is test_row_0/B:col10/1730982225009/Put/seqid=0 2024-11-07T12:23:45,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741954_1130 (size=12001) 2024-11-07T12:23:45,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e9f1cbb032064b00a069c429d14ac1ec 2024-11-07T12:23:45,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/420ddf47ff25451382c2266f84cf07f1 is 50, key is test_row_0/C:col10/1730982225009/Put/seqid=0 2024-11-07T12:23:45,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741955_1131 (size=12001) 2024-11-07T12:23:45,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/420ddf47ff25451382c2266f84cf07f1 2024-11-07T12:23:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dee275d585cd4734ae3d4ea3e0d5055a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a 2024-11-07T12:23:45,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a, entries=250, sequenceid=78, filesize=47.0 K 2024-11-07T12:23:45,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e9f1cbb032064b00a069c429d14ac1ec as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec 2024-11-07T12:23:45,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T12:23:45,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/420ddf47ff25451382c2266f84cf07f1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1 2024-11-07T12:23:45,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T12:23:45,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 258ms, sequenceid=78, compaction requested=false 2024-11-07T12:23:45,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:45,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:23:45,988 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/a57dd3586ae844ff90e969132d3327f7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/a57dd3586ae844ff90e969132d3327f7 2024-11-07T12:23:46,004 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into a57dd3586ae844ff90e969132d3327f7(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:46,004 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:46,004 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982225540; duration=0sec 2024-11-07T12:23:46,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b4b36fb5c2d47c093d76a7fb687f211_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,004 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:46,004 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:46,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741956_1132 (size=14594) 2024-11-07T12:23:46,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:46,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982286032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982286035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982286040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b4b36fb5c2d47c093d76a7fb687f211_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b4b36fb5c2d47c093d76a7fb687f211_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:46,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982286040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/071ae53c60c74b8e9c610c3e8810207c, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/071ae53c60c74b8e9c610c3e8810207c is 175, key is test_row_0/A:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982286048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741957_1133 (size=39549) 2024-11-07T12:23:46,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/071ae53c60c74b8e9c610c3e8810207c 2024-11-07T12:23:46,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/58507dd810f54e439ee6b90268266baa is 50, key is test_row_0/B:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741958_1134 (size=12001) 2024-11-07T12:23:46,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/58507dd810f54e439ee6b90268266baa 2024-11-07T12:23:46,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982286152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982286153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982286162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982286162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982286162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f252d5a1966040d8bea050abdc966d3c is 50, key is test_row_0/C:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741959_1135 (size=12001) 2024-11-07T12:23:46,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f252d5a1966040d8bea050abdc966d3c 2024-11-07T12:23:46,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/071ae53c60c74b8e9c610c3e8810207c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c 2024-11-07T12:23:46,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c, entries=200, sequenceid=93, filesize=38.6 K 2024-11-07T12:23:46,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/58507dd810f54e439ee6b90268266baa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa 2024-11-07T12:23:46,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:23:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f252d5a1966040d8bea050abdc966d3c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c 2024-11-07T12:23:46,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:23:46,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 243ms, sequenceid=93, compaction requested=true 2024-11-07T12:23:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:46,229 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:46,229 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:46,230 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:46,230 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:46,230 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,230 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=116.0 K 2024-11-07T12:23:46,230 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,230 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c] 2024-11-07T12:23:46,231 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9e587dc14a4662a318618add96c721, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223828 2024-11-07T12:23:46,232 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:46,232 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:46,232 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,232 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/a57dd3586ae844ff90e969132d3327f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.3 K 2024-11-07T12:23:46,232 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a57dd3586ae844ff90e969132d3327f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223828 2024-11-07T12:23:46,233 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting dee275d585cd4734ae3d4ea3e0d5055a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982225003 2024-11-07T12:23:46,233 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f1cbb032064b00a069c429d14ac1ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982225009 2024-11-07T12:23:46,233 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071ae53c60c74b8e9c610c3e8810207c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225657 2024-11-07T12:23:46,234 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 58507dd810f54e439ee6b90268266baa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225660 2024-11-07T12:23:46,265 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,274 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:46,275 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/b403cfc6f43645d49effaccaf4947ce9 is 50, key is test_row_0/B:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,277 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107d53ae95e52e843e5adcc0fc88f32a256_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,281 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107d53ae95e52e843e5adcc0fc88f32a256_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,281 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d53ae95e52e843e5adcc0fc88f32a256_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741960_1136 (size=12207) 2024-11-07T12:23:46,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741961_1137 (size=4469) 2024-11-07T12:23:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:46,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982286377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ce87d103c5c44b7582b7386ed1cbd900_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982226035/Put/seqid=0 2024-11-07T12:23:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982286382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982286382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982286384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982286384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741962_1138 (size=19474) 2024-11-07T12:23:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982286490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982286501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982286501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982286501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982286501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,702 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/b403cfc6f43645d49effaccaf4947ce9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b403cfc6f43645d49effaccaf4947ce9 2024-11-07T12:23:46,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982286701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T12:23:46,711 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:46,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:46,712 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into b403cfc6f43645d49effaccaf4947ce9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:46,712 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:46,712 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982226229; duration=0sec 2024-11-07T12:23:46,713 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:46,713 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T12:23:46,713 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#114 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:46,713 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:46,714 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/93f1002991f542669aa0be947566aec8 is 175, key is test_row_0/A:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:46,714 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982286711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982286711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982286711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:46,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982286712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:46,718 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:46,718 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:46,718 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,718 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0de46198431b451582c954dcd14b901b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.3 K 2024-11-07T12:23:46,719 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de46198431b451582c954dcd14b901b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730982223828 2024-11-07T12:23:46,720 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 420ddf47ff25451382c2266f84cf07f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730982225009 2024-11-07T12:23:46,720 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f252d5a1966040d8bea050abdc966d3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225660 2024-11-07T12:23:46,734 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:46,734 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/ea505ddaec5c4acf8891d5a0011e749e is 50, key is test_row_0/C:col10/1730982225660/Put/seqid=0 2024-11-07T12:23:46,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741963_1139 (size=31161) 2024-11-07T12:23:46,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/93f1002991f542669aa0be947566aec8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8 2024-11-07T12:23:46,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741964_1140 (size=12207) 2024-11-07T12:23:46,763 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into 93f1002991f542669aa0be947566aec8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:46,763 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:46,763 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982226228; duration=0sec 2024-11-07T12:23:46,775 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:46,775 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:46,779 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/ea505ddaec5c4acf8891d5a0011e749e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ea505ddaec5c4acf8891d5a0011e749e 2024-11-07T12:23:46,786 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into ea505ddaec5c4acf8891d5a0011e749e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:46,786 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:46,786 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982226229; duration=0sec 2024-11-07T12:23:46,786 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:46,786 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:46,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:46,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ce87d103c5c44b7582b7386ed1cbd900_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ce87d103c5c44b7582b7386ed1cbd900_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:46,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0da9fdd533c746be972cc4523fabb562, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:46,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0da9fdd533c746be972cc4523fabb562 is 175, key is test_row_0/A:col10/1730982226035/Put/seqid=0 2024-11-07T12:23:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:46,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741965_1141 (size=56733) 2024-11-07T12:23:46,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0da9fdd533c746be972cc4523fabb562 2024-11-07T12:23:46,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/dc5c28b868354304a6a7487f31fa2c25 is 50, key is test_row_0/B:col10/1730982226035/Put/seqid=0 2024-11-07T12:23:46,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:46,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:46,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741966_1142 (size=12001) 2024-11-07T12:23:46,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/dc5c28b868354304a6a7487f31fa2c25 2024-11-07T12:23:46,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/6e8918886ca24c3f87dd258bb051788a is 50, key is test_row_0/C:col10/1730982226035/Put/seqid=0 2024-11-07T12:23:46,894 INFO [master/8944cbb06cd2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T12:23:46,894 INFO [master/8944cbb06cd2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T12:23:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741967_1143 (size=12001) 2024-11-07T12:23:46,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/6e8918886ca24c3f87dd258bb051788a 2024-11-07T12:23:46,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/0da9fdd533c746be972cc4523fabb562 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562 2024-11-07T12:23:46,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562, entries=300, sequenceid=121, filesize=55.4 K 2024-11-07T12:23:46,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/dc5c28b868354304a6a7487f31fa2c25 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25 2024-11-07T12:23:46,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25, entries=150, sequenceid=121, filesize=11.7 K 2024-11-07T12:23:46,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/6e8918886ca24c3f87dd258bb051788a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a 2024-11-07T12:23:46,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a, entries=150, sequenceid=121, filesize=11.7 K 2024-11-07T12:23:46,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 562ms, sequenceid=121, compaction requested=false 2024-11-07T12:23:46,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:47,022 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:47,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:47,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:47,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077158a5e53f7540bf82e5369941531e3d_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741968_1144 (size=12254) 2024-11-07T12:23:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982287060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982287061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982287062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982287064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982287064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982287171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982287171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982287171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982287171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982287172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982287380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982287380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982287381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982287381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982287381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:47,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077158a5e53f7540bf82e5369941531e3d_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077158a5e53f7540bf82e5369941531e3d_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:47,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/4d503843edd44e0abd764bd8d756e262, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:47,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/4d503843edd44e0abd764bd8d756e262 is 175, key is test_row_0/A:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:47,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741969_1145 (size=31055) 2024-11-07T12:23:47,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982287691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982287692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982287692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982287692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982287692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:47,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:47,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/4d503843edd44e0abd764bd8d756e262 2024-11-07T12:23:47,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/986ae9ddfc924b80b76e803f451c9166 is 50, key is test_row_0/B:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:47,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741970_1146 (size=12101) 2024-11-07T12:23:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/986ae9ddfc924b80b76e803f451c9166 2024-11-07T12:23:47,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f3dfb2c69e004d6ba7aaab8cf0334e97 is 50, key is test_row_0/C:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:47,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741971_1147 (size=12101) 2024-11-07T12:23:47,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:47,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:47,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:48,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982288200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982288200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982288200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:48,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982288201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:48,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982288211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:48,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:48,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f3dfb2c69e004d6ba7aaab8cf0334e97 2024-11-07T12:23:48,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/4d503843edd44e0abd764bd8d756e262 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262 2024-11-07T12:23:48,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262, entries=150, sequenceid=135, filesize=30.3 K 2024-11-07T12:23:48,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/986ae9ddfc924b80b76e803f451c9166 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166 2024-11-07T12:23:48,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166, entries=150, sequenceid=135, filesize=11.8 K 2024-11-07T12:23:48,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f3dfb2c69e004d6ba7aaab8cf0334e97 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97 2024-11-07T12:23:48,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97, entries=150, sequenceid=135, filesize=11.8 K 2024-11-07T12:23:48,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1336ms, sequenceid=135, compaction requested=true 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:48,359 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:48,359 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:48,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:48,360 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:48,360 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:48,360 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:48,360 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,360 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:48,360 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,360 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=116.2 K 2024-11-07T12:23:48,360 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b403cfc6f43645d49effaccaf4947ce9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.5 K 2024-11-07T12:23:48,360 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,360 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262] 2024-11-07T12:23:48,361 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b403cfc6f43645d49effaccaf4947ce9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225660 2024-11-07T12:23:48,361 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f1002991f542669aa0be947566aec8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225660 2024-11-07T12:23:48,361 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting dc5c28b868354304a6a7487f31fa2c25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730982226035 2024-11-07T12:23:48,361 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da9fdd533c746be972cc4523fabb562, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730982226014 2024-11-07T12:23:48,361 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 986ae9ddfc924b80b76e803f451c9166, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:48,362 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d503843edd44e0abd764bd8d756e262, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:48,369 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:48,370 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:48,372 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/14b3850b37d1417fac18eefd98520aac is 50, key is test_row_0/B:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:48,372 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411074adbb00209ba4110814fb97e2eae2ace_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:48,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741972_1148 (size=12409) 2024-11-07T12:23:48,377 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411074adbb00209ba4110814fb97e2eae2ace_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:48,377 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074adbb00209ba4110814fb97e2eae2ace_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741973_1149 (size=4469) 2024-11-07T12:23:48,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T12:23:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,403 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b7c9d063d43c4ae69ef20f1f4d3398f0_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982227059/Put/seqid=0 2024-11-07T12:23:48,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741974_1150 (size=12304) 2024-11-07T12:23:48,794 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:48,794 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1f7648f5484d44ab8a1585322608cce2 is 175, key is test_row_0/A:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:48,796 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/14b3850b37d1417fac18eefd98520aac as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/14b3850b37d1417fac18eefd98520aac 2024-11-07T12:23:48,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741975_1151 (size=31363) 2024-11-07T12:23:48,803 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 14b3850b37d1417fac18eefd98520aac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:48,803 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:48,803 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982228359; duration=0sec 2024-11-07T12:23:48,803 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:48,803 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:48,804 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:48,805 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:48,805 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:48,805 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:48,805 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ea505ddaec5c4acf8891d5a0011e749e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.5 K 2024-11-07T12:23:48,806 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ea505ddaec5c4acf8891d5a0011e749e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982225660 2024-11-07T12:23:48,806 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e8918886ca24c3f87dd258bb051788a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730982226035 2024-11-07T12:23:48,806 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f3dfb2c69e004d6ba7aaab8cf0334e97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:48,813 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:48,814 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/26e1f24af40548fc99bca4f0dea32bee is 50, key is test_row_0/C:col10/1730982227020/Put/seqid=0 2024-11-07T12:23:48,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:48,821 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b7c9d063d43c4ae69ef20f1f4d3398f0_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7c9d063d43c4ae69ef20f1f4d3398f0_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8e81687fc73842b5863bbaa5b2a4b51f, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8e81687fc73842b5863bbaa5b2a4b51f is 175, key is test_row_0/A:col10/1730982227059/Put/seqid=0 2024-11-07T12:23:48,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741976_1152 (size=12409) 2024-11-07T12:23:48,835 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/26e1f24af40548fc99bca4f0dea32bee as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/26e1f24af40548fc99bca4f0dea32bee 2024-11-07T12:23:48,841 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 26e1f24af40548fc99bca4f0dea32bee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:48,841 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:48,841 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982228359; duration=0sec 2024-11-07T12:23:48,841 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:48,841 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:48,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741977_1153 (size=31105) 2024-11-07T12:23:48,843 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8e81687fc73842b5863bbaa5b2a4b51f 2024-11-07T12:23:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/35751724d78b4ce3b4dd4de618b2a96c is 50, key is test_row_0/B:col10/1730982227059/Put/seqid=0 2024-11-07T12:23:48,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741978_1154 (size=12151) 2024-11-07T12:23:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:49,204 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1f7648f5484d44ab8a1585322608cce2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2 2024-11-07T12:23:49,209 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into 1f7648f5484d44ab8a1585322608cce2(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:49,209 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:49,209 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982228359; duration=0sec 2024-11-07T12:23:49,209 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:49,209 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:49,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982289220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982289220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,226 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,227 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982289225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982289227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982289227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,259 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/35751724d78b4ce3b4dd4de618b2a96c 2024-11-07T12:23:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/4d4360f557d74296bb4ae6f8135fba86 is 50, key is test_row_0/C:col10/1730982227059/Put/seqid=0 2024-11-07T12:23:49,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741979_1155 (size=12151) 2024-11-07T12:23:49,287 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/4d4360f557d74296bb4ae6f8135fba86 2024-11-07T12:23:49,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8e81687fc73842b5863bbaa5b2a4b51f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f 2024-11-07T12:23:49,300 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f, entries=150, sequenceid=161, filesize=30.4 K 2024-11-07T12:23:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/35751724d78b4ce3b4dd4de618b2a96c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c 2024-11-07T12:23:49,320 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T12:23:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/4d4360f557d74296bb4ae6f8135fba86 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86 2024-11-07T12:23:49,329 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T12:23:49,330 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 927ms, sequenceid=161, compaction requested=false 2024-11-07T12:23:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T12:23:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T12:23:49,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T12:23:49,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-11-07T12:23:49,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6230 sec 2024-11-07T12:23:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:49,344 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:49,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:49,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071c3cd6a5bf114e2b95c2042749eb31a4_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741980_1156 (size=17284) 2024-11-07T12:23:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982289387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982289390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982289390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982289501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982289501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982289501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982289711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982289711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982289711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:49,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:49,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071c3cd6a5bf114e2b95c2042749eb31a4_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071c3cd6a5bf114e2b95c2042749eb31a4_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:49,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/615591a4df1d4d45807cf42364bce111, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:49,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/615591a4df1d4d45807cf42364bce111 is 175, key is test_row_0/A:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:49,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741981_1157 (size=48389) 2024-11-07T12:23:50,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982290020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/615591a4df1d4d45807cf42364bce111 2024-11-07T12:23:50,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/516413cbc53e44b8b950da86bcf80b62 is 50, key is test_row_0/B:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:50,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741982_1158 (size=12151) 2024-11-07T12:23:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982290531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982290531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982290531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:50,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/516413cbc53e44b8b950da86bcf80b62 2024-11-07T12:23:50,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5c1fbbcdf1a6488db7736e754c1f053c is 50, key is test_row_0/C:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:50,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741983_1159 (size=12151) 2024-11-07T12:23:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T12:23:50,871 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:50,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T12:23:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:50,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:50,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:50,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:51,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5c1fbbcdf1a6488db7736e754c1f053c 2024-11-07T12:23:51,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/615591a4df1d4d45807cf42364bce111 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111 2024-11-07T12:23:51,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111, entries=250, sequenceid=177, filesize=47.3 K 2024-11-07T12:23:51,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/516413cbc53e44b8b950da86bcf80b62 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62 2024-11-07T12:23:51,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62, entries=150, sequenceid=177, filesize=11.9 K 2024-11-07T12:23:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5c1fbbcdf1a6488db7736e754c1f053c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c 2024-11-07T12:23:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-07T12:23:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1676ms, sequenceid=177, compaction requested=true 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:51,021 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:23:51,021 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:51,022 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:51,022 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:51,022 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:51,022 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:51,022 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:51,022 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:51,023 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=108.3 K 2024-11-07T12:23:51,023 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/14b3850b37d1417fac18eefd98520aac, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.9 K 2024-11-07T12:23:51,023 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:51,023 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111] 2024-11-07T12:23:51,023 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b3850b37d1417fac18eefd98520aac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:51,023 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7648f5484d44ab8a1585322608cce2, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:51,023 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 35751724d78b4ce3b4dd4de618b2a96c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730982227059 2024-11-07T12:23:51,023 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e81687fc73842b5863bbaa5b2a4b51f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730982227059 2024-11-07T12:23:51,024 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615591a4df1d4d45807cf42364bce111, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:51,024 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 516413cbc53e44b8b950da86bcf80b62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:51,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T12:23:51,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:51,026 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:23:51,031 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a728529200df405b95e33ca7be42c074_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982229371/Put/seqid=0 2024-11-07T12:23:51,038 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:51,038 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/2eb5e44aee73495da8c5b2853a7cd1c7 is 50, key is test_row_0/B:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:51,041 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b885c3bb84f14a5395069411244a1c9a_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:51,042 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b885c3bb84f14a5395069411244a1c9a_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:51,042 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b885c3bb84f14a5395069411244a1c9a_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:51,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741984_1160 (size=12304) 2024-11-07T12:23:51,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741985_1161 (size=12561) 2024-11-07T12:23:51,067 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/2eb5e44aee73495da8c5b2853a7cd1c7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/2eb5e44aee73495da8c5b2853a7cd1c7 2024-11-07T12:23:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741986_1162 (size=4469) 2024-11-07T12:23:51,072 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#132 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:51,073 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a0bee14c43b24f1a92af35b070ecb89b is 175, key is test_row_0/A:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:51,074 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 2eb5e44aee73495da8c5b2853a7cd1c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:51,074 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:51,074 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982231021; duration=0sec 2024-11-07T12:23:51,075 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:51,075 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:51,075 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:51,077 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:51,077 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:51,077 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:51,077 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/26e1f24af40548fc99bca4f0dea32bee, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=35.9 K 2024-11-07T12:23:51,077 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e1f24af40548fc99bca4f0dea32bee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730982226380 2024-11-07T12:23:51,078 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4360f557d74296bb4ae6f8135fba86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730982227059 2024-11-07T12:23:51,078 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1fbbcdf1a6488db7736e754c1f053c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:51,087 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:51,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741987_1163 (size=31515) 2024-11-07T12:23:51,087 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/a066d3b7fe1647dbbac15b6f062cf18d is 50, key is test_row_0/C:col10/1730982229344/Put/seqid=0 2024-11-07T12:23:51,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741988_1164 (size=12561) 2024-11-07T12:23:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:51,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982291260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982291261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982291371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982291371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:51,464 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a728529200df405b95e33ca7be42c074_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a728529200df405b95e33ca7be42c074_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:51,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/329a3ecd878c4fa3ba61b96f4bb762af, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:51,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/329a3ecd878c4fa3ba61b96f4bb762af is 175, key is test_row_0/A:col10/1730982229371/Put/seqid=0 2024-11-07T12:23:51,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741989_1165 (size=31105) 2024-11-07T12:23:51,497 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a0bee14c43b24f1a92af35b070ecb89b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b 2024-11-07T12:23:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:51,501 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into a0bee14c43b24f1a92af35b070ecb89b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:51,501 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:51,501 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982231021; duration=0sec 2024-11-07T12:23:51,501 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:51,501 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:51,508 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/a066d3b7fe1647dbbac15b6f062cf18d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a066d3b7fe1647dbbac15b6f062cf18d 2024-11-07T12:23:51,514 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into a066d3b7fe1647dbbac15b6f062cf18d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:51,514 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:51,514 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982231021; duration=0sec 2024-11-07T12:23:51,514 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:51,514 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:51,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982291551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982291551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,553 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:51,553 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982291551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,555 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982291580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982291580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/329a3ecd878c4fa3ba61b96f4bb762af 2024-11-07T12:23:51,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e66cc55e46bc401dbb68a3f6ab9f4cbc is 50, key is test_row_0/B:col10/1730982229371/Put/seqid=0 2024-11-07T12:23:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741990_1166 (size=12151) 2024-11-07T12:23:51,890 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e66cc55e46bc401dbb68a3f6ab9f4cbc 2024-11-07T12:23:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982291891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982291892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af0f88a735234f0fbedb57f6022bd508 is 50, key is test_row_0/C:col10/1730982229371/Put/seqid=0 2024-11-07T12:23:51,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741991_1167 (size=12151) 2024-11-07T12:23:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:52,313 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af0f88a735234f0fbedb57f6022bd508 2024-11-07T12:23:52,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/329a3ecd878c4fa3ba61b96f4bb762af as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af 2024-11-07T12:23:52,324 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af, entries=150, sequenceid=200, filesize=30.4 K 2024-11-07T12:23:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/e66cc55e46bc401dbb68a3f6ab9f4cbc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc 2024-11-07T12:23:52,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T12:23:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af0f88a735234f0fbedb57f6022bd508 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508 2024-11-07T12:23:52,346 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508, entries=150, sequenceid=200, filesize=11.9 K 2024-11-07T12:23:52,347 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1321ms, sequenceid=200, compaction requested=false 2024-11-07T12:23:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:52,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T12:23:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T12:23:52,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T12:23:52,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-11-07T12:23:52,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4800 sec 2024-11-07T12:23:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:52,402 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:52,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:23:52,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107655ac8b140d648a2a17f87a2be9d7884_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:52,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741992_1168 (size=14794) 2024-11-07T12:23:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982292474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982292478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982292590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982292591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982292800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982292800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:52,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:52,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107655ac8b140d648a2a17f87a2be9d7884_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107655ac8b140d648a2a17f87a2be9d7884_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:52,841 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/eb05bd4cf9c64d4d982db7c4a1c6738d, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:52,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/eb05bd4cf9c64d4d982db7c4a1c6738d is 175, key is test_row_0/A:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:52,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741993_1169 (size=39749) 2024-11-07T12:23:52,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/eb05bd4cf9c64d4d982db7c4a1c6738d 2024-11-07T12:23:52,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c440b5d6b417460590ae17794ca679bc is 50, key is test_row_0/B:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:52,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741994_1170 (size=12151) 2024-11-07T12:23:52,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c440b5d6b417460590ae17794ca679bc 2024-11-07T12:23:52,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af08b8993ca34dfb9521643406245463 is 50, key is test_row_0/C:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741995_1171 (size=12151) 2024-11-07T12:23:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T12:23:53,021 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:53,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T12:23:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:23:53,023 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:53,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:53,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982293110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982293110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:23:53,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:23:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af08b8993ca34dfb9521643406245463 2024-11-07T12:23:53,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/eb05bd4cf9c64d4d982db7c4a1c6738d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d 2024-11-07T12:23:53,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d, entries=200, sequenceid=218, filesize=38.8 K 2024-11-07T12:23:53,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c440b5d6b417460590ae17794ca679bc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc 2024-11-07T12:23:53,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc, entries=150, sequenceid=218, filesize=11.9 K 2024-11-07T12:23:53,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/af08b8993ca34dfb9521643406245463 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463 2024-11-07T12:23:53,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463, entries=150, sequenceid=218, filesize=11.9 K 2024-11-07T12:23:53,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 906ms, sequenceid=218, compaction requested=true 2024-11-07T12:23:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:53,308 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:53,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:53,309 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:53,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:53,310 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:53,310 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:53,310 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:53,310 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:53,310 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,310 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,310 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/2eb5e44aee73495da8c5b2853a7cd1c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.0 K 2024-11-07T12:23:53,310 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=100.0 K 2024-11-07T12:23:53,310 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,310 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d] 2024-11-07T12:23:53,311 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb5e44aee73495da8c5b2853a7cd1c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:53,311 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0bee14c43b24f1a92af35b070ecb89b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:53,311 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e66cc55e46bc401dbb68a3f6ab9f4cbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730982229371 2024-11-07T12:23:53,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 329a3ecd878c4fa3ba61b96f4bb762af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730982229371 2024-11-07T12:23:53,312 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c440b5d6b417460590ae17794ca679bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:53,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb05bd4cf9c64d4d982db7c4a1c6738d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:53,319 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,326 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:53,327 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/81a962dc71ac4f2eaf3db6ed7fa2d26a is 50, key is test_row_0/B:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:53,328 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110766747e91866d41268204c443a6d5bca9_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T12:23:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,330 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:23:53,330 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110766747e91866d41268204c443a6d5bca9_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,330 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110766747e91866d41268204c443a6d5bca9_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:23:53,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741997_1173 (size=4469) 2024-11-07T12:23:53,345 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#141 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:53,345 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/bd0411c188cc44c09629fcfb7c2e97ad is 175, key is test_row_0/A:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:53,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741996_1172 (size=12663) 2024-11-07T12:23:53,357 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/81a962dc71ac4f2eaf3db6ed7fa2d26a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/81a962dc71ac4f2eaf3db6ed7fa2d26a 2024-11-07T12:23:53,362 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 81a962dc71ac4f2eaf3db6ed7fa2d26a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:53,363 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:53,363 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982233309; duration=0sec 2024-11-07T12:23:53,363 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:53,363 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:53,363 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da7290506cdb4bc6b304ba851d9a7ecc_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982232468/Put/seqid=0 2024-11-07T12:23:53,364 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:53,364 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:53,364 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,365 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a066d3b7fe1647dbbac15b6f062cf18d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.0 K 2024-11-07T12:23:53,365 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a066d3b7fe1647dbbac15b6f062cf18d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1730982229221 2024-11-07T12:23:53,366 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting af0f88a735234f0fbedb57f6022bd508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1730982229371 2024-11-07T12:23:53,367 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting af08b8993ca34dfb9521643406245463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:53,380 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:53,380 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/0b4024374d4e47688e5a9b994d2b3e8c is 50, key is test_row_0/C:col10/1730982231255/Put/seqid=0 2024-11-07T12:23:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741998_1174 (size=31617) 2024-11-07T12:23:53,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741999_1175 (size=12304) 2024-11-07T12:23:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,402 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da7290506cdb4bc6b304ba851d9a7ecc_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7290506cdb4bc6b304ba851d9a7ecc_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d9868557867340e4bfe9b421ebc097e1, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d9868557867340e4bfe9b421ebc097e1 is 175, key is test_row_0/A:col10/1730982232468/Put/seqid=0 2024-11-07T12:23:53,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742001_1177 (size=31105) 2024-11-07T12:23:53,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742000_1176 (size=12663) 2024-11-07T12:23:53,429 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d9868557867340e4bfe9b421ebc097e1 2024-11-07T12:23:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 is 50, key is test_row_0/B:col10/1730982232468/Put/seqid=0 2024-11-07T12:23:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742002_1178 (size=12151) 2024-11-07T12:23:53,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 2024-11-07T12:23:53,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1d4ee3c0b0094331a213fbfc10084a2d is 50, key is test_row_0/C:col10/1730982232468/Put/seqid=0 2024-11-07T12:23:53,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742003_1179 (size=12151) 2024-11-07T12:23:53,525 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1d4ee3c0b0094331a213fbfc10084a2d 2024-11-07T12:23:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d9868557867340e4bfe9b421ebc097e1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1 2024-11-07T12:23:53,538 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1, entries=150, sequenceid=239, filesize=30.4 K 2024-11-07T12:23:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 2024-11-07T12:23:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,544 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:23:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1d4ee3c0b0094331a213fbfc10084a2d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d 2024-11-07T12:23:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,549 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:23:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,550 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 220ms, sequenceid=239, compaction requested=false 2024-11-07T12:23:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T12:23:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T12:23:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T12:23:53,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 528 msec 2024-11-07T12:23:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 533 msec 2024-11-07T12:23:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:53,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:53,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T12:23:53,651 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T12:23:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:53,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107717b896218e24255a3d02997f0e50484_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:53,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742005_1181 (size=24758) 2024-11-07T12:23:53,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:53,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107717b896218e24255a3d02997f0e50484_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107717b896218e24255a3d02997f0e50484_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a4d21a3121be43cb8a0b51e52c08b59e, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982293708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a4d21a3121be43cb8a0b51e52c08b59e is 175, key is test_row_0/A:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742004_1180 (size=74394) 2024-11-07T12:23:53,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a4d21a3121be43cb8a0b51e52c08b59e 2024-11-07T12:23:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982293710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982293711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982293712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982293715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/095a7fe803f84130a6df2f0f5b616db9 is 50, key is test_row_0/B:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:53,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742006_1182 (size=12151) 2024-11-07T12:23:53,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/095a7fe803f84130a6df2f0f5b616db9 2024-11-07T12:23:53,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/bd2a96f7c7074147973166295d1acf35 is 50, key is test_row_0/C:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:53,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742007_1183 (size=12151) 2024-11-07T12:23:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:53,788 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/bd0411c188cc44c09629fcfb7c2e97ad as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad 2024-11-07T12:23:53,794 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into bd0411c188cc44c09629fcfb7c2e97ad(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:53,794 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:53,794 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982233308; duration=0sec 2024-11-07T12:23:53,794 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:53,794 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:53,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:23:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982293820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,826 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/0b4024374d4e47688e5a9b994d2b3e8c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0b4024374d4e47688e5a9b994d2b3e8c 2024-11-07T12:23:53,833 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 0b4024374d4e47688e5a9b994d2b3e8c(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:53,833 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:53,833 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982233309; duration=0sec 2024-11-07T12:23:53,833 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,833 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982293830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982293831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982293831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982293831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:53,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:23:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:53,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:54,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982294030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982294041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982294041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982294041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982294041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:23:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:54,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/bd2a96f7c7074147973166295d1acf35 2024-11-07T12:23:54,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/a4d21a3121be43cb8a0b51e52c08b59e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e 2024-11-07T12:23:54,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e, entries=400, sequenceid=251, filesize=72.7 K 2024-11-07T12:23:54,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/095a7fe803f84130a6df2f0f5b616db9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9 2024-11-07T12:23:54,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:23:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/bd2a96f7c7074147973166295d1acf35 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35 2024-11-07T12:23:54,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:23:54,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 532ms, sequenceid=251, compaction requested=true 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:54,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:54,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:54,176 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137116 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:54,176 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:54,177 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,177 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,177 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=133.9 K 2024-11-07T12:23:54,177 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/81a962dc71ac4f2eaf3db6ed7fa2d26a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.1 K 2024-11-07T12:23:54,177 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e] 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 81a962dc71ac4f2eaf3db6ed7fa2d26a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0411c188cc44c09629fcfb7c2e97ad, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:54,177 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 73eedfe3cc2e4aa989e63e7e8cf8e3b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982232468 2024-11-07T12:23:54,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9868557867340e4bfe9b421ebc097e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982232468 2024-11-07T12:23:54,178 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 095a7fe803f84130a6df2f0f5b616db9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233622 2024-11-07T12:23:54,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d21a3121be43cb8a0b51e52c08b59e, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233619 2024-11-07T12:23:54,198 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:54,198 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/851347eedadd4d8085e6f438053129c9 is 50, key is test_row_0/B:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:54,199 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:54,213 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c1e25aa5b1064ff18ababa21945f5608_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:54,216 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c1e25aa5b1064ff18ababa21945f5608_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:54,216 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c1e25aa5b1064ff18ababa21945f5608_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:54,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742008_1184 (size=12765) 2024-11-07T12:23:54,248 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/851347eedadd4d8085e6f438053129c9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/851347eedadd4d8085e6f438053129c9 2024-11-07T12:23:54,256 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 851347eedadd4d8085e6f438053129c9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:54,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:54,256 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982234175; duration=0sec 2024-11-07T12:23:54,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:54,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:54,256 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:54,257 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:54,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:54,258 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,258 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0b4024374d4e47688e5a9b994d2b3e8c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.1 K 2024-11-07T12:23:54,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4024374d4e47688e5a9b994d2b3e8c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730982231255 2024-11-07T12:23:54,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4ee3c0b0094331a213fbfc10084a2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982232468 2024-11-07T12:23:54,259 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2a96f7c7074147973166295d1acf35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233622 2024-11-07T12:23:54,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742009_1185 (size=4469) 2024-11-07T12:23:54,263 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#151 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:54,264 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8d040804f3104fb7b11c83a5c6fc69bb is 175, key is test_row_0/A:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:54,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T12:23:54,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:23:54,269 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:54,270 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/90e75632fb2d4ea79036b68786285ee8 is 50, key is test_row_0/C:col10/1730982233642/Put/seqid=0 2024-11-07T12:23:54,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742010_1186 (size=31719) 2024-11-07T12:23:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:54,287 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/8d040804f3104fb7b11c83a5c6fc69bb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb 2024-11-07T12:23:54,293 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into 8d040804f3104fb7b11c83a5c6fc69bb(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:54,293 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:54,293 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982234175; duration=0sec 2024-11-07T12:23:54,293 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:54,293 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4a2595078a341ceb8240fe4ee0ff7fa_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982233711/Put/seqid=0 2024-11-07T12:23:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742011_1187 (size=12765) 2024-11-07T12:23:54,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742012_1188 (size=12454) 2024-11-07T12:23:54,327 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/90e75632fb2d4ea79036b68786285ee8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/90e75632fb2d4ea79036b68786285ee8 2024-11-07T12:23:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:54,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4a2595078a341ceb8240fe4ee0ff7fa_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4a2595078a341ceb8240fe4ee0ff7fa_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d4fb9a537cda4a518cd17e8eae638a4a, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d4fb9a537cda4a518cd17e8eae638a4a is 175, key is test_row_0/A:col10/1730982233711/Put/seqid=0 2024-11-07T12:23:54,337 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 90e75632fb2d4ea79036b68786285ee8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:54,337 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:54,337 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982234175; duration=0sec 2024-11-07T12:23:54,337 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:54,337 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:54,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742013_1189 (size=31255) 2024-11-07T12:23:54,344 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d4fb9a537cda4a518cd17e8eae638a4a 2024-11-07T12:23:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982294356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982294357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982294358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982294358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/b443b8db3d2e428c985afa6b294f64f3 is 50, key is test_row_0/B:col10/1730982233711/Put/seqid=0 2024-11-07T12:23:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982294361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742014_1190 (size=12301) 2024-11-07T12:23:54,383 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/b443b8db3d2e428c985afa6b294f64f3 2024-11-07T12:23:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5bd1119be57b44ff97c24f25787f48d2 is 50, key is test_row_0/C:col10/1730982233711/Put/seqid=0 2024-11-07T12:23:54,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742015_1191 (size=12301) 2024-11-07T12:23:54,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982294470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982294470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982294470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982294471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982294471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982294680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982294681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982294681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982294681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982294681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:54,809 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5bd1119be57b44ff97c24f25787f48d2 2024-11-07T12:23:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d4fb9a537cda4a518cd17e8eae638a4a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a 2024-11-07T12:23:54,819 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a, entries=150, sequenceid=279, filesize=30.5 K 2024-11-07T12:23:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/b443b8db3d2e428c985afa6b294f64f3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3 2024-11-07T12:23:54,826 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:23:54,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5bd1119be57b44ff97c24f25787f48d2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2 2024-11-07T12:23:54,832 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:23:54,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 566ms, sequenceid=279, compaction requested=false 2024-11-07T12:23:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T12:23:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T12:23:54,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T12:23:54,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1800 sec 2024-11-07T12:23:54,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1860 sec 2024-11-07T12:23:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:54,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:54,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:23:55,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710a9a38bec1c46488fe82f8f8d4f51c2_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982295020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982295021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982295022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982295022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982295026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742016_1192 (size=12454) 2024-11-07T12:23:55,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:55,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710a9a38bec1c46488fe82f8f8d4f51c2_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710a9a38bec1c46488fe82f8f8d4f51c2_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:55,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ab0e7bd2d24a454f87ab0404292312d4, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ab0e7bd2d24a454f87ab0404292312d4 is 175, key is test_row_0/A:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742017_1193 (size=31255) 2024-11-07T12:23:55,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ab0e7bd2d24a454f87ab0404292312d4 2024-11-07T12:23:55,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/874cea04909045ca9a20239672a225d4 is 50, key is test_row_0/B:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742018_1194 (size=12301) 2024-11-07T12:23:55,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/874cea04909045ca9a20239672a225d4 2024-11-07T12:23:55,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f2e45fe842ca42cda50388893b7e9bf6 is 50, key is test_row_0/C:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742019_1195 (size=12301) 2024-11-07T12:23:55,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f2e45fe842ca42cda50388893b7e9bf6 2024-11-07T12:23:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ab0e7bd2d24a454f87ab0404292312d4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4 2024-11-07T12:23:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982295131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982295131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982295131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982295131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4, entries=150, sequenceid=294, filesize=30.5 K 2024-11-07T12:23:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982295131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/874cea04909045ca9a20239672a225d4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4 2024-11-07T12:23:55,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:23:55,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/f2e45fe842ca42cda50388893b7e9bf6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6 2024-11-07T12:23:55,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T12:23:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 155ms, sequenceid=294, compaction requested=true 2024-11-07T12:23:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:55,149 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:55,149 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:55,150 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:55,150 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:55,150 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:55,150 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:55,150 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:55,150 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:55,150 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=92.0 K 2024-11-07T12:23:55,150 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/851347eedadd4d8085e6f438053129c9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.5 K 2024-11-07T12:23:55,150 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:55,150 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4] 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 851347eedadd4d8085e6f438053129c9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233622 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d040804f3104fb7b11c83a5c6fc69bb, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233622 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b443b8db3d2e428c985afa6b294f64f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982233694 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4fb9a537cda4a518cd17e8eae638a4a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982233694 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0e7bd2d24a454f87ab0404292312d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:55,151 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 874cea04909045ca9a20239672a225d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:55,173 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,176 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:55,176 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/f9efda212d7e484ca1133fd64e0a2bcf is 50, key is test_row_0/B:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077ad917a9085748969368beaa925a6299_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,180 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077ad917a9085748969368beaa925a6299_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,180 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077ad917a9085748969368beaa925a6299_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742020_1196 (size=13017) 2024-11-07T12:23:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742021_1197 (size=4469) 2024-11-07T12:23:55,198 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#159 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:55,199 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d221c154083245a7b382798a868efc47 is 175, key is test_row_0/A:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742022_1198 (size=31971) 2024-11-07T12:23:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:55,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:55,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:23:55,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110703dead8ee96b435aadd7380fbbe86ec8_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982235349/Put/seqid=0 2024-11-07T12:23:55,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982295368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982295372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982295375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982295375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982295378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742023_1199 (size=12454) 2024-11-07T12:23:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982295483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982295483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982295491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982295491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982295491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,602 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/f9efda212d7e484ca1133fd64e0a2bcf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f9efda212d7e484ca1133fd64e0a2bcf 2024-11-07T12:23:55,609 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into f9efda212d7e484ca1133fd64e0a2bcf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:55,609 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:55,609 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982235149; duration=0sec 2024-11-07T12:23:55,609 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:55,609 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:55,609 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:23:55,611 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:23:55,611 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:55,611 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:55,611 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/90e75632fb2d4ea79036b68786285ee8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=36.5 K 2024-11-07T12:23:55,612 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e75632fb2d4ea79036b68786285ee8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982233622 2024-11-07T12:23:55,612 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd1119be57b44ff97c24f25787f48d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982233694 2024-11-07T12:23:55,613 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e45fe842ca42cda50388893b7e9bf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:55,626 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:55,627 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/6605357d8bb945539f03ee62848fc5e5 is 50, key is test_row_0/C:col10/1730982234357/Put/seqid=0 2024-11-07T12:23:55,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742024_1200 (size=13017) 2024-11-07T12:23:55,648 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/6605357d8bb945539f03ee62848fc5e5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6605357d8bb945539f03ee62848fc5e5 2024-11-07T12:23:55,657 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 6605357d8bb945539f03ee62848fc5e5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:55,657 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:55,657 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982235149; duration=0sec 2024-11-07T12:23:55,658 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:55,658 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:55,660 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/d221c154083245a7b382798a868efc47 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47 2024-11-07T12:23:55,668 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into d221c154083245a7b382798a868efc47(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:55,668 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:55,668 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982235148; duration=0sec 2024-11-07T12:23:55,669 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:55,669 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:55,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982295690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982295690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982295701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982295701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982295701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:55,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:55,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T12:23:55,803 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:55,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:55,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110703dead8ee96b435aadd7380fbbe86ec8_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703dead8ee96b435aadd7380fbbe86ec8_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T12:23:55,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/96a91daed48e452bb89b3b90a50ceba8, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/96a91daed48e452bb89b3b90a50ceba8 is 175, key is test_row_0/A:col10/1730982235349/Put/seqid=0 2024-11-07T12:23:55,807 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:23:55,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:55,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742025_1201 (size=31255) 2024-11-07T12:23:55,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/96a91daed48e452bb89b3b90a50ceba8 2024-11-07T12:23:55,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/31ba0fbb5a164cdc841fbf7c2bd675f0 is 50, key is test_row_0/B:col10/1730982235349/Put/seqid=0 2024-11-07T12:23:55,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742026_1202 (size=12301) 2024-11-07T12:23:55,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/31ba0fbb5a164cdc841fbf7c2bd675f0 2024-11-07T12:23:55,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/cf916e53018342d384ec3732d813cb34 is 50, key is test_row_0/C:col10/1730982235349/Put/seqid=0 2024-11-07T12:23:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742027_1203 (size=12301) 2024-11-07T12:23:55,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/cf916e53018342d384ec3732d813cb34 2024-11-07T12:23:55,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/96a91daed48e452bb89b3b90a50ceba8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8 2024-11-07T12:23:55,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8, entries=150, sequenceid=319, filesize=30.5 K 2024-11-07T12:23:55,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/31ba0fbb5a164cdc841fbf7c2bd675f0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0 2024-11-07T12:23:55,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T12:23:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/cf916e53018342d384ec3732d813cb34 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34 2024-11-07T12:23:55,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T12:23:55,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 553ms, sequenceid=319, compaction requested=false 2024-11-07T12:23:55,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:23:55,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T12:23:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:55,963 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:23:55,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107dbf8dea29bf8481f9db03609fc4f137e_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982235374/Put/seqid=0 2024-11-07T12:23:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742028_1204 (size=12454) 2024-11-07T12:23:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:55,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107dbf8dea29bf8481f9db03609fc4f137e_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dbf8dea29bf8481f9db03609fc4f137e_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/6d9fd448f30842a6865f2e2940031b9c, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/6d9fd448f30842a6865f2e2940031b9c is 175, key is test_row_0/A:col10/1730982235374/Put/seqid=0 2024-11-07T12:23:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:56,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:56,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742029_1205 (size=31255) 2024-11-07T12:23:56,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/6d9fd448f30842a6865f2e2940031b9c 2024-11-07T12:23:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c61bf2861fba4fe4a4d3640a44de386e is 50, key is test_row_0/B:col10/1730982235374/Put/seqid=0 2024-11-07T12:23:56,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742030_1206 (size=12301) 2024-11-07T12:23:56,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982296037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982296038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982296038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982296039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982296040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:23:56,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982296151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982296151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982296151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982296151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982296161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982296361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982296361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982296361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982296361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982296370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,427 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c61bf2861fba4fe4a4d3640a44de386e 2024-11-07T12:23:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:23:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/7683a92bbad0454abbae68724f55b0eb is 50, key is test_row_0/C:col10/1730982235374/Put/seqid=0 2024-11-07T12:23:56,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742031_1207 (size=12301) 2024-11-07T12:23:56,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/7683a92bbad0454abbae68724f55b0eb 2024-11-07T12:23:56,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/6d9fd448f30842a6865f2e2940031b9c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c 2024-11-07T12:23:56,480 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c, entries=150, sequenceid=333, filesize=30.5 K 2024-11-07T12:23:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/c61bf2861fba4fe4a4d3640a44de386e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e 2024-11-07T12:23:56,485 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-07T12:23:56,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/7683a92bbad0454abbae68724f55b0eb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb 2024-11-07T12:23:56,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb, entries=150, sequenceid=333, filesize=12.0 K 2024-11-07T12:23:56,494 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 532ms, sequenceid=333, compaction requested=true 2024-11-07T12:23:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T12:23:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T12:23:56,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T12:23:56,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 687 msec 2024-11-07T12:23:56,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 696 msec 2024-11-07T12:23:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:56,674 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:56,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:23:56,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982296679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982296681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982296682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982296682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982296680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110785f838383192426288a81b922ccac414_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:56,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742032_1208 (size=17534) 2024-11-07T12:23:56,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982296790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982296790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982296791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982296791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T12:23:56,941 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:56,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T12:23:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:56,943 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:56,944 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:56,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:57,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982297000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982297001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982297001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982297001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:57,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:57,115 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110785f838383192426288a81b922ccac414_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785f838383192426288a81b922ccac414_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:57,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/800e2eabd192414d985dcf4b70a7ac53, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:57,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/800e2eabd192414d985dcf4b70a7ac53 is 175, key is test_row_0/A:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:57,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742033_1209 (size=48639) 2024-11-07T12:23:57,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/800e2eabd192414d985dcf4b70a7ac53 2024-11-07T12:23:57,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/fcdac101db5f490681fea85f06358315 is 50, key is test_row_0/B:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:57,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742034_1210 (size=12301) 2024-11-07T12:23:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982297191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982297311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982297311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982297312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982297312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/fcdac101db5f490681fea85f06358315 2024-11-07T12:23:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:57,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/b7e0c6132cc0422aaef714f859c98673 is 50, key is test_row_0/C:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742035_1211 (size=12301) 2024-11-07T12:23:57,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982297820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982297820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982297820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982297821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:57,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:57,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/b7e0c6132cc0422aaef714f859c98673 2024-11-07T12:23:57,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/800e2eabd192414d985dcf4b70a7ac53 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53 2024-11-07T12:23:57,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53, entries=250, sequenceid=360, filesize=47.5 K 2024-11-07T12:23:57,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/fcdac101db5f490681fea85f06358315 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315 2024-11-07T12:23:57,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:23:57,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/b7e0c6132cc0422aaef714f859c98673 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673 2024-11-07T12:23:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:57,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T12:23:57,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1322ms, sequenceid=360, compaction requested=true 2024-11-07T12:23:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:23:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:57,996 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:23:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:57,996 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:57,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:23:57,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:57,998 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:57,999 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:23:57,999 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,999 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f9efda212d7e484ca1133fd64e0a2bcf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=48.8 K 2024-11-07T12:23:57,999 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:57,999 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:23:57,999 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:57,999 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=139.8 K 2024-11-07T12:23:57,999 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:58,000 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53] 2024-11-07T12:23:58,000 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f9efda212d7e484ca1133fd64e0a2bcf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:58,000 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d221c154083245a7b382798a868efc47, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:58,000 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 31ba0fbb5a164cdc841fbf7c2bd675f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730982235020 2024-11-07T12:23:58,001 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a91daed48e452bb89b3b90a50ceba8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730982235020 2024-11-07T12:23:58,001 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c61bf2861fba4fe4a4d3640a44de386e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1730982235359 2024-11-07T12:23:58,001 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d9fd448f30842a6865f2e2940031b9c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1730982235359 2024-11-07T12:23:58,001 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fcdac101db5f490681fea85f06358315, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236039 2024-11-07T12:23:58,001 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800e2eabd192414d985dcf4b70a7ac53, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236035 2024-11-07T12:23:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,009 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,022 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070adc5be18ae2477eb0b6a0868032b423_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,023 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#171 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,024 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/bf519d37a9f644fbb268e4210864dcfb is 50, key is test_row_0/B:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,043 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070adc5be18ae2477eb0b6a0868032b423_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,043 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070adc5be18ae2477eb0b6a0868032b423_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742036_1212 (size=13153) 2024-11-07T12:23:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,052 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/bf519d37a9f644fbb268e4210864dcfb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/bf519d37a9f644fbb268e4210864dcfb 2024-11-07T12:23:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,059 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into bf519d37a9f644fbb268e4210864dcfb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:58,059 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:58,059 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=12, startTime=1730982237996; duration=0sec 2024-11-07T12:23:58,059 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:23:58,059 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:23:58,059 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:23:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,060 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:23:58,061 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:23:58,061 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,061 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6605357d8bb945539f03ee62848fc5e5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=48.8 K 2024-11-07T12:23:58,061 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6605357d8bb945539f03ee62848fc5e5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730982234357 2024-11-07T12:23:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,062 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting cf916e53018342d384ec3732d813cb34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730982235020 2024-11-07T12:23:58,062 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 7683a92bbad0454abbae68724f55b0eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1730982235359 2024-11-07T12:23:58,062 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e0c6132cc0422aaef714f859c98673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236039 2024-11-07T12:23:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742037_1213 (size=4469) 2024-11-07T12:23:58,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T12:23:58,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:58,067 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:23:58,070 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#172 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:58,070 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/f1ffe89c3ade472e82ab2f6410bd2185 is 175, key is test_row_0/A:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:58,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742038_1214 (size=32107) 2024-11-07T12:23:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107990e9aad821547048c739504a5c5fe88_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982236681/Put/seqid=0 2024-11-07T12:23:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,086 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#174 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:23:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,086 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/433a25c96016464f94818a6764a713f1 is 50, key is test_row_0/C:col10/1730982236673/Put/seqid=0 2024-11-07T12:23:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742039_1215 (size=9914) 2024-11-07T12:23:58,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,105 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107990e9aad821547048c739504a5c5fe88_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107990e9aad821547048c739504a5c5fe88_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:58,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/00569f7c1d59470ba0cdc6c6361d5471, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/00569f7c1d59470ba0cdc6c6361d5471 is 175, key is test_row_0/A:col10/1730982236681/Put/seqid=0 2024-11-07T12:23:58,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742040_1216 (size=13153) 2024-11-07T12:23:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742041_1217 (size=22561) 2024-11-07T12:23:58,120 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/00569f7c1d59470ba0cdc6c6361d5471 2024-11-07T12:23:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/17351d45b80d40f7acc9b63b4163fe2d is 50, key is test_row_0/B:col10/1730982236681/Put/seqid=0 2024-11-07T12:23:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742042_1218 (size=9857) 2024-11-07T12:23:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/17351d45b80d40f7acc9b63b4163fe2d 2024-11-07T12:23:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/534479ead587453f87c8108bb75ee9e9 is 50, key is test_row_0/C:col10/1730982236681/Put/seqid=0 2024-11-07T12:23:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742043_1219 (size=9857) 2024-11-07T12:23:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:58,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982298358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982298471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,492 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/f1ffe89c3ade472e82ab2f6410bd2185 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185 2024-11-07T12:23:58,499 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into f1ffe89c3ade472e82ab2f6410bd2185(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:58,499 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:58,499 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=12, startTime=1730982237996; duration=0sec 2024-11-07T12:23:58,499 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:58,499 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:23:58,522 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/433a25c96016464f94818a6764a713f1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/433a25c96016464f94818a6764a713f1 2024-11-07T12:23:58,529 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 433a25c96016464f94818a6764a713f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:23:58,529 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:58,529 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=12, startTime=1730982237996; duration=0sec 2024-11-07T12:23:58,529 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:23:58,529 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:23:58,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/534479ead587453f87c8108bb75ee9e9 2024-11-07T12:23:58,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/00569f7c1d59470ba0cdc6c6361d5471 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471 2024-11-07T12:23:58,574 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471, entries=100, sequenceid=370, filesize=22.0 K 2024-11-07T12:23:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/17351d45b80d40f7acc9b63b4163fe2d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d 2024-11-07T12:23:58,583 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d, entries=100, sequenceid=370, filesize=9.6 K 2024-11-07T12:23:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/534479ead587453f87c8108bb75ee9e9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9 2024-11-07T12:23:58,592 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9, entries=100, sequenceid=370, filesize=9.6 K 2024-11-07T12:23:58,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 526ms, sequenceid=370, compaction requested=false 2024-11-07T12:23:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:23:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T12:23:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T12:23:58,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T12:23:58,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6500 sec 2024-11-07T12:23:58,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6570 sec 2024-11-07T12:23:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:23:58,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:23:58,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e18d6465573446aacef04832887cc24_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982238684/Put/seqid=0 2024-11-07T12:23:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982298719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742044_1220 (size=12454) 2024-11-07T12:23:58,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982298831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982298831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,832 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982298841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982298841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982298841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:58,842 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,842 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:58,842 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982299040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T12:23:59,091 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:23:59,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:23:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T12:23:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:23:59,093 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:23:59,094 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:23:59,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:23:59,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:23:59,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e18d6465573446aacef04832887cc24_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e18d6465573446aacef04832887cc24_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:23:59,168 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1c52294f39b941c9a2283e3f10191d4d, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:23:59,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1c52294f39b941c9a2283e3f10191d4d is 175, key is test_row_0/A:col10/1730982238684/Put/seqid=0 2024-11-07T12:23:59,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742045_1221 (size=31255) 2024-11-07T12:23:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:23:59,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:23:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982299351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:59,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:23:59,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:23:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,576 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1c52294f39b941c9a2283e3f10191d4d 2024-11-07T12:23:59,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/373ead1dc45b4569b4498524be7e1b91 is 50, key is test_row_0/B:col10/1730982238684/Put/seqid=0 2024-11-07T12:23:59,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742046_1222 (size=12301) 2024-11-07T12:23:59,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:23:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:23:59,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:23:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:23:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:23:59,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:23:59,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:23:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982299861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:23:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:23:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:23:59,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/373ead1dc45b4569b4498524be7e1b91 2024-11-07T12:23:59,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1faef4a3e3894b3a9e3767ed2e3cfbfa is 50, key is test_row_0/C:col10/1730982238684/Put/seqid=0 2024-11-07T12:23:59,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742047_1223 (size=12301) 2024-11-07T12:24:00,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:24:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:24:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:24:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:24:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:24:00,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:24:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:24:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:00,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1faef4a3e3894b3a9e3767ed2e3cfbfa 2024-11-07T12:24:00,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/1c52294f39b941c9a2283e3f10191d4d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d 2024-11-07T12:24:00,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d, entries=150, sequenceid=400, filesize=30.5 K 2024-11-07T12:24:00,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/373ead1dc45b4569b4498524be7e1b91 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91 2024-11-07T12:24:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91, entries=150, sequenceid=400, filesize=12.0 K 2024-11-07T12:24:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/1faef4a3e3894b3a9e3767ed2e3cfbfa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa 2024-11-07T12:24:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa, entries=150, sequenceid=400, filesize=12.0 K 2024-11-07T12:24:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1719ms, sequenceid=400, compaction requested=true 2024-11-07T12:24:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:00,420 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:00,420 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:00,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:24:00,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:00,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:24:00,421 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,422 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,422 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/bf519d37a9f644fbb268e4210864dcfb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=34.5 K 2024-11-07T12:24:00,422 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=83.9 K 2024-11-07T12:24:00,422 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,422 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d] 2024-11-07T12:24:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bf519d37a9f644fbb268e4210864dcfb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236039 2024-11-07T12:24:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,425 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ffe89c3ade472e82ab2f6410bd2185, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236039 2024-11-07T12:24:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 17351d45b80d40f7acc9b63b4163fe2d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730982236680 2024-11-07T12:24:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00569f7c1d59470ba0cdc6c6361d5471, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730982236680 2024-11-07T12:24:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c52294f39b941c9a2283e3f10191d4d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:00,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 373ead1dc45b4569b4498524be7e1b91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,436 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,437 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,438 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/06eaa09c68e049ab9eb89aeac5210da3 is 50, key is test_row_0/B:col10/1730982238684/Put/seqid=0 2024-11-07T12:24:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,445 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110732809241890643b9b85697f4bf7a00b6_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,446 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110732809241890643b9b85697f4bf7a00b6_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:00,446 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110732809241890643b9b85697f4bf7a00b6_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742049_1225 (size=4469) 2024-11-07T12:24:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742048_1224 (size=13255) 2024-11-07T12:24:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,464 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#181 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,464 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/36a85262c48f45808ea27e308bb75e25 is 175, key is test_row_0/A:col10/1730982238684/Put/seqid=0 2024-11-07T12:24:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742050_1226 (size=32209) 2024-11-07T12:24:00,469 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/06eaa09c68e049ab9eb89aeac5210da3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/06eaa09c68e049ab9eb89aeac5210da3 2024-11-07T12:24:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T12:24:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,470 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:24:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,478 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/36a85262c48f45808ea27e308bb75e25 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25 2024-11-07T12:24:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,480 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 06eaa09c68e049ab9eb89aeac5210da3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:00,480 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:00,480 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982240420; duration=0sec 2024-11-07T12:24:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,480 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:00,480 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:24:00,480 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,481 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,482 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:24:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,482 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:00,482 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/433a25c96016464f94818a6764a713f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=34.5 K 2024-11-07T12:24:00,482 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 433a25c96016464f94818a6764a713f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730982236039 2024-11-07T12:24:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,483 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 534479ead587453f87c8108bb75ee9e9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730982236680 2024-11-07T12:24:00,483 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1faef4a3e3894b3a9e3767ed2e3cfbfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,486 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into 36a85262c48f45808ea27e308bb75e25(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:00,486 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:00,486 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982240420; duration=0sec 2024-11-07T12:24:00,486 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:00,486 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:24:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107be47ae447f704113b2477fa408b64bb7_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982238709/Put/seqid=0 2024-11-07T12:24:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:24:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,494 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,494 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/e9e68f1384be4ce0815fd26211a248e6 is 50, key is test_row_0/C:col10/1730982238684/Put/seqid=0 2024-11-07T12:24:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742051_1227 (size=13255) 2024-11-07T12:24:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742052_1228 (size=9914) 2024-11-07T12:24:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. as already flushing 2024-11-07T12:24:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:00,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982300910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982300912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:00,918 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/e9e68f1384be4ce0815fd26211a248e6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/e9e68f1384be4ce0815fd26211a248e6 2024-11-07T12:24:00,920 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107be47ae447f704113b2477fa408b64bb7_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107be47ae447f704113b2477fa408b64bb7_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dc923f0e833e494a800ad5121e06714f, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:00,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982300915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dc923f0e833e494a800ad5121e06714f is 175, key is test_row_0/A:col10/1730982238709/Put/seqid=0 2024-11-07T12:24:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982300915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982300918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:00,929 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into e9e68f1384be4ce0815fd26211a248e6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:00,929 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:00,930 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982240420; duration=0sec 2024-11-07T12:24:00,930 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:00,930 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:24:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742053_1229 (size=22561) 2024-11-07T12:24:00,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dc923f0e833e494a800ad5121e06714f 2024-11-07T12:24:00,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/f587f87311da4dc0ac7c93dd1644cdf9 is 50, key is test_row_0/B:col10/1730982238709/Put/seqid=0 2024-11-07T12:24:00,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742054_1230 (size=9857) 2024-11-07T12:24:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982301020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982301021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982301031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982301031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982301031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982301231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982301231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:24:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982301241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982301241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982301241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,367 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/f587f87311da4dc0ac7c93dd1644cdf9 2024-11-07T12:24:01,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/42231ac2e98f4eaaa7567f0c79f2295e is 50, key is test_row_0/C:col10/1730982238709/Put/seqid=0 2024-11-07T12:24:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742055_1231 (size=9857) 2024-11-07T12:24:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982301541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982301541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982301551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982301551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982301551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:01,781 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/42231ac2e98f4eaaa7567f0c79f2295e 2024-11-07T12:24:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/dc923f0e833e494a800ad5121e06714f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f 2024-11-07T12:24:01,792 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f, entries=100, sequenceid=408, filesize=22.0 K 2024-11-07T12:24:01,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/f587f87311da4dc0ac7c93dd1644cdf9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9 2024-11-07T12:24:01,797 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9, entries=100, sequenceid=408, filesize=9.6 K 2024-11-07T12:24:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/42231ac2e98f4eaaa7567f0c79f2295e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e 2024-11-07T12:24:01,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e, entries=100, sequenceid=408, filesize=9.6 K 2024-11-07T12:24:01,803 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1333ms, sequenceid=408, compaction requested=false 2024-11-07T12:24:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:01,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T12:24:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T12:24:01,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T12:24:01,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7110 sec 2024-11-07T12:24:01,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7150 sec 2024-11-07T12:24:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:02,054 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:02,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-07T12:24:02,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107446a2cf2fb4f437e83e9788dd696db2b_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982302058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982302060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982302060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982302060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982302063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742056_1232 (size=17534) 2024-11-07T12:24:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982302170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982302170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982302381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982302381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:02,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107446a2cf2fb4f437e83e9788dd696db2b_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107446a2cf2fb4f437e83e9788dd696db2b_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:02,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ef8b9690acbc4b8ea7758960f20b770c, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:02,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ef8b9690acbc4b8ea7758960f20b770c is 175, key is test_row_0/A:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742057_1233 (size=48639) 2024-11-07T12:24:02,550 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-07T12:24:02,550 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:02,550 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,550 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,550 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-07T12:24:02,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:02,550 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:02,550 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,550 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:02,556 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-07T12:24:02,556 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:02,556 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,556 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:02,556 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-07T12:24:02,556 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:02,556 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,557 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:02,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:02,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982302691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982302691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:02,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ef8b9690acbc4b8ea7758960f20b770c 2024-11-07T12:24:02,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/82121fb189494c889eabeaa0573f8e90 is 50, key is test_row_0/B:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:02,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742058_1234 (size=12301) 2024-11-07T12:24:03,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56030 deadline: 1730982303071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:03,071 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56014 deadline: 1730982303081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1730982303081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:03,082 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:03,082 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 548e342ecfd588a5f02a3dbe0c2bf2ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56012 deadline: 1730982303201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55988 deadline: 1730982303201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:03,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T12:24:03,251 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:03,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/82121fb189494c889eabeaa0573f8e90 2024-11-07T12:24:03,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5a2688907b1f43969b23f6f45c36a800 is 50, key is test_row_0/C:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:03,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742059_1235 (size=12301) 2024-11-07T12:24:03,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5a2688907b1f43969b23f6f45c36a800 2024-11-07T12:24:03,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/ef8b9690acbc4b8ea7758960f20b770c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c 2024-11-07T12:24:03,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c, entries=250, sequenceid=441, filesize=47.5 K 2024-11-07T12:24:03,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/82121fb189494c889eabeaa0573f8e90 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90 2024-11-07T12:24:03,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90, entries=150, sequenceid=441, filesize=12.0 K 2024-11-07T12:24:03,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/5a2688907b1f43969b23f6f45c36a800 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800 2024-11-07T12:24:03,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800, entries=150, sequenceid=441, filesize=12.0 K 2024-11-07T12:24:03,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1695ms, sequenceid=441, compaction requested=true 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:03,750 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:03,750 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 548e342ecfd588a5f02a3dbe0c2bf2ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:03,751 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:03,751 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:03,751 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/B is initiating minor compaction (all files) 2024-11-07T12:24:03,751 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/A is initiating minor compaction (all files) 2024-11-07T12:24:03,751 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/B in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:03,751 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/A in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:03,751 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/06eaa09c68e049ab9eb89aeac5210da3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=34.6 K 2024-11-07T12:24:03,751 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=101.0 K 2024-11-07T12:24:03,751 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:03,751 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c] 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 06eaa09c68e049ab9eb89aeac5210da3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a85262c48f45808ea27e308bb75e25, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f587f87311da4dc0ac7c93dd1644cdf9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1730982238709 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc923f0e833e494a800ad5121e06714f, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1730982238709 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 82121fb189494c889eabeaa0573f8e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730982240912 2024-11-07T12:24:03,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef8b9690acbc4b8ea7758960f20b770c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730982240912 2024-11-07T12:24:03,760 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:03,761 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:03,762 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/89994a9b4e464df291f32ac15d9dd1de is 50, key is test_row_0/B:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:03,769 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110763b3e86b453f4bf184a1a5a1967809e2_548e342ecfd588a5f02a3dbe0c2bf2ae store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:03,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742060_1236 (size=13357) 2024-11-07T12:24:03,791 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/89994a9b4e464df291f32ac15d9dd1de as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/89994a9b4e464df291f32ac15d9dd1de 2024-11-07T12:24:03,797 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110763b3e86b453f4bf184a1a5a1967809e2_548e342ecfd588a5f02a3dbe0c2bf2ae, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:03,797 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110763b3e86b453f4bf184a1a5a1967809e2_548e342ecfd588a5f02a3dbe0c2bf2ae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:03,799 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/B of 548e342ecfd588a5f02a3dbe0c2bf2ae into 89994a9b4e464df291f32ac15d9dd1de(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:03,799 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:03,799 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/B, priority=13, startTime=1730982243750; duration=0sec 2024-11-07T12:24:03,799 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:03,799 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:B 2024-11-07T12:24:03,799 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:03,801 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:03,801 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 548e342ecfd588a5f02a3dbe0c2bf2ae/C is initiating minor compaction (all files) 2024-11-07T12:24:03,801 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 548e342ecfd588a5f02a3dbe0c2bf2ae/C in TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:03,801 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/e9e68f1384be4ce0815fd26211a248e6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp, totalSize=34.6 K 2024-11-07T12:24:03,802 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e9e68f1384be4ce0815fd26211a248e6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1730982238343 2024-11-07T12:24:03,802 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 42231ac2e98f4eaaa7567f0c79f2295e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1730982238709 2024-11-07T12:24:03,803 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2688907b1f43969b23f6f45c36a800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1730982240912 2024-11-07T12:24:03,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742061_1237 (size=4469) 2024-11-07T12:24:03,821 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#A#compaction#190 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:03,822 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/311aa6cf28624a489d4b6fc1416b6960 is 175, key is test_row_0/A:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:03,827 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 548e342ecfd588a5f02a3dbe0c2bf2ae#C#compaction#191 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:03,828 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/4c52ad7582ab414a9d0792cb8c7ea6cb is 50, key is test_row_0/C:col10/1730982240912/Put/seqid=0 2024-11-07T12:24:03,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742062_1238 (size=32311) 2024-11-07T12:24:03,839 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/311aa6cf28624a489d4b6fc1416b6960 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/311aa6cf28624a489d4b6fc1416b6960 2024-11-07T12:24:03,845 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/A of 548e342ecfd588a5f02a3dbe0c2bf2ae into 311aa6cf28624a489d4b6fc1416b6960(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:03,845 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:03,845 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/A, priority=13, startTime=1730982243750; duration=0sec 2024-11-07T12:24:03,845 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:03,845 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:A 2024-11-07T12:24:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742063_1239 (size=13357) 2024-11-07T12:24:03,860 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/4c52ad7582ab414a9d0792cb8c7ea6cb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4c52ad7582ab414a9d0792cb8c7ea6cb 2024-11-07T12:24:03,874 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 548e342ecfd588a5f02a3dbe0c2bf2ae/C of 548e342ecfd588a5f02a3dbe0c2bf2ae into 4c52ad7582ab414a9d0792cb8c7ea6cb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:03,874 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:03,874 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae., storeName=548e342ecfd588a5f02a3dbe0c2bf2ae/C, priority=13, startTime=1730982243750; duration=0sec 2024-11-07T12:24:03,874 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:03,874 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 548e342ecfd588a5f02a3dbe0c2bf2ae:C 2024-11-07T12:24:04,214 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-11-07T12:24:04,214 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:04,214 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:04,214 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:04,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:04,217 INFO [Thread-583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-583. 2024-11-07T12:24:04,217 DEBUG [Thread-583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:04,217 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:04,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:04,218 DEBUG [Thread-583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,082 INFO [Thread-575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-575. 2024-11-07T12:24:05,082 DEBUG [Thread-575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:05,082 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,082 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:05,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:05,102 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-11-07T12:24:05,102 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:05,102 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 548e342ecfd588a5f02a3dbe0c2bf2ae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:24:05,103 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-11-07T12:24:05,103 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:05,103 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:05,103 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,103 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3593 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3523 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1553 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4658 rows 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1560 2024-11-07T12:24:05,103 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4679 rows 2024-11-07T12:24:05,103 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:24:05,104 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:24:05,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:05,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:05,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:24:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:05,108 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982245108"}]},"ts":"1730982245108"} 2024-11-07T12:24:05,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd790deae89f428996d1beb9febd6eb8_548e342ecfd588a5f02a3dbe0c2bf2ae is 50, key is test_row_0/A:col10/1730982245101/Put/seqid=0 2024-11-07T12:24:05,110 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:24:05,110 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:24:05,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:24:05,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, UNASSIGN}] 2024-11-07T12:24:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742064_1240 (size=12454) 2024-11-07T12:24:05,114 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, UNASSIGN 2024-11-07T12:24:05,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:05,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, UNASSIGN because future has completed 2024-11-07T12:24:05,117 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:24:05,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:05,269 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:05,269 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:24:05,269 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 548e342ecfd588a5f02a3dbe0c2bf2ae, disabling compactions & flushes 2024-11-07T12:24:05,269 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:05,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:05,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd790deae89f428996d1beb9febd6eb8_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd790deae89f428996d1beb9febd6eb8_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:05,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fbc11e84f97045d19fbf4006689f43dc, store: [table=TestAcidGuarantees family=A region=548e342ecfd588a5f02a3dbe0c2bf2ae] 2024-11-07T12:24:05,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fbc11e84f97045d19fbf4006689f43dc is 175, key is test_row_0/A:col10/1730982245101/Put/seqid=0 2024-11-07T12:24:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742065_1241 (size=31255) 2024-11-07T12:24:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:05,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fbc11e84f97045d19fbf4006689f43dc 2024-11-07T12:24:05,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/197c72a95d1f46a183f38d71b7155a6f is 50, key is test_row_0/B:col10/1730982245101/Put/seqid=0 2024-11-07T12:24:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742066_1242 (size=12301) 2024-11-07T12:24:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:06,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/197c72a95d1f46a183f38d71b7155a6f 2024-11-07T12:24:06,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/ce733860f7e347b79536058a8a15c810 is 50, key is test_row_0/C:col10/1730982245101/Put/seqid=0 2024-11-07T12:24:06,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742067_1243 (size=12301) 2024-11-07T12:24:06,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/ce733860f7e347b79536058a8a15c810 2024-11-07T12:24:06,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/A/fbc11e84f97045d19fbf4006689f43dc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fbc11e84f97045d19fbf4006689f43dc 2024-11-07T12:24:06,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fbc11e84f97045d19fbf4006689f43dc, entries=150, sequenceid=455, filesize=30.5 K 2024-11-07T12:24:06,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/B/197c72a95d1f46a183f38d71b7155a6f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/197c72a95d1f46a183f38d71b7155a6f 2024-11-07T12:24:06,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/197c72a95d1f46a183f38d71b7155a6f, entries=150, sequenceid=455, filesize=12.0 K 2024-11-07T12:24:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/.tmp/C/ce733860f7e347b79536058a8a15c810 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ce733860f7e347b79536058a8a15c810 2024-11-07T12:24:06,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ce733860f7e347b79536058a8a15c810, entries=150, sequenceid=455, filesize=12.0 K 2024-11-07T12:24:06,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 548e342ecfd588a5f02a3dbe0c2bf2ae in 1686ms, sequenceid=455, compaction requested=false 2024-11-07T12:24:06,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: 2024-11-07T12:24:06,788 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:06,788 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:06,788 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. after waiting 0 ms 2024-11-07T12:24:06,788 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c] to archive 2024-11-07T12:24:06,789 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:06,790 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/cb6cb436a0cd49538e865fe09839be91 2024-11-07T12:24:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/3a93f6ffe9ab4e798763488faa2a4939 2024-11-07T12:24:06,792 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0ab99efd8fe94bd9bfaf7360798f7587 2024-11-07T12:24:06,793 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fe9e587dc14a4662a318618add96c721 2024-11-07T12:24:06,794 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dee275d585cd4734ae3d4ea3e0d5055a 2024-11-07T12:24:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/071ae53c60c74b8e9c610c3e8810207c 2024-11-07T12:24:06,796 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/93f1002991f542669aa0be947566aec8 2024-11-07T12:24:06,797 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/0da9fdd533c746be972cc4523fabb562 2024-11-07T12:24:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1f7648f5484d44ab8a1585322608cce2 2024-11-07T12:24:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/4d503843edd44e0abd764bd8d756e262 2024-11-07T12:24:06,800 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8e81687fc73842b5863bbaa5b2a4b51f 2024-11-07T12:24:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/615591a4df1d4d45807cf42364bce111 2024-11-07T12:24:06,802 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a0bee14c43b24f1a92af35b070ecb89b 2024-11-07T12:24:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/329a3ecd878c4fa3ba61b96f4bb762af 2024-11-07T12:24:06,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/eb05bd4cf9c64d4d982db7c4a1c6738d 2024-11-07T12:24:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/bd0411c188cc44c09629fcfb7c2e97ad 2024-11-07T12:24:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d9868557867340e4bfe9b421ebc097e1 2024-11-07T12:24:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/a4d21a3121be43cb8a0b51e52c08b59e 2024-11-07T12:24:06,808 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/8d040804f3104fb7b11c83a5c6fc69bb 2024-11-07T12:24:06,809 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d4fb9a537cda4a518cd17e8eae638a4a 2024-11-07T12:24:06,810 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/d221c154083245a7b382798a868efc47 2024-11-07T12:24:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ab0e7bd2d24a454f87ab0404292312d4 2024-11-07T12:24:06,812 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/96a91daed48e452bb89b3b90a50ceba8 2024-11-07T12:24:06,813 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/6d9fd448f30842a6865f2e2940031b9c 2024-11-07T12:24:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/800e2eabd192414d985dcf4b70a7ac53 2024-11-07T12:24:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/f1ffe89c3ade472e82ab2f6410bd2185 2024-11-07T12:24:06,817 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/00569f7c1d59470ba0cdc6c6361d5471 2024-11-07T12:24:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/36a85262c48f45808ea27e308bb75e25 2024-11-07T12:24:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/1c52294f39b941c9a2283e3f10191d4d 2024-11-07T12:24:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/dc923f0e833e494a800ad5121e06714f 2024-11-07T12:24:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/ef8b9690acbc4b8ea7758960f20b770c 2024-11-07T12:24:06,822 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/a57dd3586ae844ff90e969132d3327f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b403cfc6f43645d49effaccaf4947ce9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/14b3850b37d1417fac18eefd98520aac, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/2eb5e44aee73495da8c5b2853a7cd1c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/81a962dc71ac4f2eaf3db6ed7fa2d26a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/851347eedadd4d8085e6f438053129c9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f9efda212d7e484ca1133fd64e0a2bcf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/bf519d37a9f644fbb268e4210864dcfb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/06eaa09c68e049ab9eb89aeac5210da3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90] to archive 2024-11-07T12:24:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/0aadec8cb9994c128c5d022414b7ddbc 2024-11-07T12:24:06,826 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/535f327f6491422c8d194f9f9f864986 2024-11-07T12:24:06,827 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/a57dd3586ae844ff90e969132d3327f7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/a57dd3586ae844ff90e969132d3327f7 2024-11-07T12:24:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/25ef12b9dee440b29b9ec1a9baa6c5fd 2024-11-07T12:24:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e9f1cbb032064b00a069c429d14ac1ec 2024-11-07T12:24:06,830 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b403cfc6f43645d49effaccaf4947ce9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b403cfc6f43645d49effaccaf4947ce9 2024-11-07T12:24:06,831 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/58507dd810f54e439ee6b90268266baa 2024-11-07T12:24:06,832 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/dc5c28b868354304a6a7487f31fa2c25 2024-11-07T12:24:06,833 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/14b3850b37d1417fac18eefd98520aac to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/14b3850b37d1417fac18eefd98520aac 2024-11-07T12:24:06,834 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/986ae9ddfc924b80b76e803f451c9166 2024-11-07T12:24:06,835 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/35751724d78b4ce3b4dd4de618b2a96c 2024-11-07T12:24:06,836 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/2eb5e44aee73495da8c5b2853a7cd1c7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/2eb5e44aee73495da8c5b2853a7cd1c7 2024-11-07T12:24:06,838 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/516413cbc53e44b8b950da86bcf80b62 2024-11-07T12:24:06,839 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/e66cc55e46bc401dbb68a3f6ab9f4cbc 2024-11-07T12:24:06,840 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/81a962dc71ac4f2eaf3db6ed7fa2d26a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/81a962dc71ac4f2eaf3db6ed7fa2d26a 2024-11-07T12:24:06,841 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c440b5d6b417460590ae17794ca679bc 2024-11-07T12:24:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/73eedfe3cc2e4aa989e63e7e8cf8e3b3 2024-11-07T12:24:06,843 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/851347eedadd4d8085e6f438053129c9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/851347eedadd4d8085e6f438053129c9 2024-11-07T12:24:06,844 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/095a7fe803f84130a6df2f0f5b616db9 2024-11-07T12:24:06,845 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/b443b8db3d2e428c985afa6b294f64f3 2024-11-07T12:24:06,847 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f9efda212d7e484ca1133fd64e0a2bcf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f9efda212d7e484ca1133fd64e0a2bcf 2024-11-07T12:24:06,848 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/874cea04909045ca9a20239672a225d4 2024-11-07T12:24:06,849 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/31ba0fbb5a164cdc841fbf7c2bd675f0 2024-11-07T12:24:06,850 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/c61bf2861fba4fe4a4d3640a44de386e 2024-11-07T12:24:06,851 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/bf519d37a9f644fbb268e4210864dcfb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/bf519d37a9f644fbb268e4210864dcfb 2024-11-07T12:24:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/fcdac101db5f490681fea85f06358315 2024-11-07T12:24:06,853 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/17351d45b80d40f7acc9b63b4163fe2d 2024-11-07T12:24:06,854 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/06eaa09c68e049ab9eb89aeac5210da3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/06eaa09c68e049ab9eb89aeac5210da3 2024-11-07T12:24:06,855 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/373ead1dc45b4569b4498524be7e1b91 2024-11-07T12:24:06,856 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/f587f87311da4dc0ac7c93dd1644cdf9 2024-11-07T12:24:06,857 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/82121fb189494c889eabeaa0573f8e90 2024-11-07T12:24:06,859 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0de46198431b451582c954dcd14b901b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ea505ddaec5c4acf8891d5a0011e749e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/26e1f24af40548fc99bca4f0dea32bee, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a066d3b7fe1647dbbac15b6f062cf18d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0b4024374d4e47688e5a9b994d2b3e8c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/90e75632fb2d4ea79036b68786285ee8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6605357d8bb945539f03ee62848fc5e5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/433a25c96016464f94818a6764a713f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/e9e68f1384be4ce0815fd26211a248e6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800] to archive 2024-11-07T12:24:06,860 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:06,861 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a4c831cb789549f1a879df01bc1e6719 2024-11-07T12:24:06,862 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/baa5fb0fcb6644e09e2787ca0fc1ebbf 2024-11-07T12:24:06,864 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0de46198431b451582c954dcd14b901b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0de46198431b451582c954dcd14b901b 2024-11-07T12:24:06,865 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f9e566636b3d40cc80c3f426def9ca88 2024-11-07T12:24:06,866 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/420ddf47ff25451382c2266f84cf07f1 2024-11-07T12:24:06,867 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ea505ddaec5c4acf8891d5a0011e749e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ea505ddaec5c4acf8891d5a0011e749e 2024-11-07T12:24:06,868 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f252d5a1966040d8bea050abdc966d3c 2024-11-07T12:24:06,870 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6e8918886ca24c3f87dd258bb051788a 2024-11-07T12:24:06,871 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/26e1f24af40548fc99bca4f0dea32bee to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/26e1f24af40548fc99bca4f0dea32bee 2024-11-07T12:24:06,872 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f3dfb2c69e004d6ba7aaab8cf0334e97 2024-11-07T12:24:06,874 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4d4360f557d74296bb4ae6f8135fba86 2024-11-07T12:24:06,875 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a066d3b7fe1647dbbac15b6f062cf18d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/a066d3b7fe1647dbbac15b6f062cf18d 2024-11-07T12:24:06,876 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5c1fbbcdf1a6488db7736e754c1f053c 2024-11-07T12:24:06,877 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af0f88a735234f0fbedb57f6022bd508 2024-11-07T12:24:06,879 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0b4024374d4e47688e5a9b994d2b3e8c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/0b4024374d4e47688e5a9b994d2b3e8c 2024-11-07T12:24:06,880 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/af08b8993ca34dfb9521643406245463 2024-11-07T12:24:06,881 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1d4ee3c0b0094331a213fbfc10084a2d 2024-11-07T12:24:06,882 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/90e75632fb2d4ea79036b68786285ee8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/90e75632fb2d4ea79036b68786285ee8 2024-11-07T12:24:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/bd2a96f7c7074147973166295d1acf35 2024-11-07T12:24:06,884 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5bd1119be57b44ff97c24f25787f48d2 2024-11-07T12:24:06,886 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6605357d8bb945539f03ee62848fc5e5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/6605357d8bb945539f03ee62848fc5e5 2024-11-07T12:24:06,887 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/f2e45fe842ca42cda50388893b7e9bf6 2024-11-07T12:24:06,889 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/cf916e53018342d384ec3732d813cb34 2024-11-07T12:24:06,890 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/7683a92bbad0454abbae68724f55b0eb 2024-11-07T12:24:06,891 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/433a25c96016464f94818a6764a713f1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/433a25c96016464f94818a6764a713f1 2024-11-07T12:24:06,893 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/b7e0c6132cc0422aaef714f859c98673 2024-11-07T12:24:06,894 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/534479ead587453f87c8108bb75ee9e9 2024-11-07T12:24:06,895 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/e9e68f1384be4ce0815fd26211a248e6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/e9e68f1384be4ce0815fd26211a248e6 2024-11-07T12:24:06,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/1faef4a3e3894b3a9e3767ed2e3cfbfa 2024-11-07T12:24:06,898 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/42231ac2e98f4eaaa7567f0c79f2295e 2024-11-07T12:24:06,900 DEBUG [StoreCloser-TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/5a2688907b1f43969b23f6f45c36a800 2024-11-07T12:24:06,919 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=4 2024-11-07T12:24:06,920 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae. 2024-11-07T12:24:06,920 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 548e342ecfd588a5f02a3dbe0c2bf2ae: Waiting for close lock at 1730982245269Running coprocessor pre-close hooks at 1730982245269Disabling compacts and flushes for region at 1730982245269Disabling writes for close at 1730982246788 (+1519 ms)Writing region close event to WAL at 1730982246916 (+128 ms)Running coprocessor post-close hooks at 1730982246920 (+4 ms)Closed at 1730982246920 2024-11-07T12:24:06,922 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:06,923 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=548e342ecfd588a5f02a3dbe0c2bf2ae, regionState=CLOSED 2024-11-07T12:24:06,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:06,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T12:24:06,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 548e342ecfd588a5f02a3dbe0c2bf2ae, server=8944cbb06cd2,46221,1730982182218 in 1.8080 sec 2024-11-07T12:24:06,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T12:24:06,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=548e342ecfd588a5f02a3dbe0c2bf2ae, UNASSIGN in 1.8140 sec 2024-11-07T12:24:06,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T12:24:06,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8170 sec 2024-11-07T12:24:06,931 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982246931"}]},"ts":"1730982246931"} 2024-11-07T12:24:06,933 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:24:06,933 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:24:06,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8290 sec 2024-11-07T12:24:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T12:24:07,251 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:07,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:24:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,253 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:24:07,254 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,256 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits] 2024-11-07T12:24:07,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/311aa6cf28624a489d4b6fc1416b6960 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/311aa6cf28624a489d4b6fc1416b6960 2024-11-07T12:24:07,266 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fbc11e84f97045d19fbf4006689f43dc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/A/fbc11e84f97045d19fbf4006689f43dc 2024-11-07T12:24:07,269 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/197c72a95d1f46a183f38d71b7155a6f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/197c72a95d1f46a183f38d71b7155a6f 2024-11-07T12:24:07,270 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/89994a9b4e464df291f32ac15d9dd1de to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/B/89994a9b4e464df291f32ac15d9dd1de 2024-11-07T12:24:07,276 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4c52ad7582ab414a9d0792cb8c7ea6cb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/4c52ad7582ab414a9d0792cb8c7ea6cb 2024-11-07T12:24:07,283 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ce733860f7e347b79536058a8a15c810 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/C/ce733860f7e347b79536058a8a15c810 2024-11-07T12:24:07,290 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits/458.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae/recovered.edits/458.seqid 2024-11-07T12:24:07,291 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,291 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:24:07,292 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:24:07,293 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:24:07,301 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703dead8ee96b435aadd7380fbbe86ec8_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110703dead8ee96b435aadd7380fbbe86ec8_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,302 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710a9a38bec1c46488fe82f8f8d4f51c2_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710a9a38bec1c46488fe82f8f8d4f51c2_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,304 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071c3cd6a5bf114e2b95c2042749eb31a4_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071c3cd6a5bf114e2b95c2042749eb31a4_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,305 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e18d6465573446aacef04832887cc24_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071e18d6465573446aacef04832887cc24_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,306 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b4b36fb5c2d47c093d76a7fb687f211_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b4b36fb5c2d47c093d76a7fb687f211_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,308 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107446a2cf2fb4f437e83e9788dd696db2b_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107446a2cf2fb4f437e83e9788dd696db2b_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,309 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107655ac8b140d648a2a17f87a2be9d7884_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107655ac8b140d648a2a17f87a2be9d7884_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,310 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077158a5e53f7540bf82e5369941531e3d_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077158a5e53f7540bf82e5369941531e3d_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,312 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107717b896218e24255a3d02997f0e50484_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107717b896218e24255a3d02997f0e50484_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,313 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b534dbb6804b72a563c7b63a9cb496_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b534dbb6804b72a563c7b63a9cb496_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,314 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077fa4b157addd43a38fdb772ec2ca11dd_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077fa4b157addd43a38fdb772ec2ca11dd_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,316 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785f838383192426288a81b922ccac414_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785f838383192426288a81b922ccac414_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,318 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107990e9aad821547048c739504a5c5fe88_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107990e9aad821547048c739504a5c5fe88_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,321 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a728529200df405b95e33ca7be42c074_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a728529200df405b95e33ca7be42c074_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,322 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b238e2803aa24fa2917c72b56caa38d5_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b238e2803aa24fa2917c72b56caa38d5_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,325 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7c9d063d43c4ae69ef20f1f4d3398f0_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7c9d063d43c4ae69ef20f1f4d3398f0_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,326 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107be47ae447f704113b2477fa408b64bb7_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107be47ae447f704113b2477fa408b64bb7_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,328 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4a2595078a341ceb8240fe4ee0ff7fa_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4a2595078a341ceb8240fe4ee0ff7fa_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,329 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd790deae89f428996d1beb9febd6eb8_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd790deae89f428996d1beb9febd6eb8_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,331 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ce87d103c5c44b7582b7386ed1cbd900_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ce87d103c5c44b7582b7386ed1cbd900_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,332 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3e86f1c4ec14148bba0a5de17531840_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3e86f1c4ec14148bba0a5de17531840_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,334 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7290506cdb4bc6b304ba851d9a7ecc_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da7290506cdb4bc6b304ba851d9a7ecc_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,335 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dbf8dea29bf8481f9db03609fc4f137e_548e342ecfd588a5f02a3dbe0c2bf2ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dbf8dea29bf8481f9db03609fc4f137e_548e342ecfd588a5f02a3dbe0c2bf2ae 2024-11-07T12:24:07,335 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:24:07,337 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,341 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:24:07,343 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:24:07,345 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,345 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:24:07,345 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982247345"}]},"ts":"9223372036854775807"} 2024-11-07T12:24:07,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:24:07,348 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 548e342ecfd588a5f02a3dbe0c2bf2ae, NAME => 'TestAcidGuarantees,,1730982219392.548e342ecfd588a5f02a3dbe0c2bf2ae.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:24:07,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:24:07,349 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982247349"}]},"ts":"9223372036854775807"} 2024-11-07T12:24:07,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:24:07,353 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-07T12:24:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T12:24:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:24:07,362 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:07,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1393209540_22 at /127.0.0.1:47672 [Waiting for operation #721] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_514357573_22 at /127.0.0.1:35388 [Waiting for operation #534] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1393209540_22 at /127.0.0.1:35382 [Waiting for operation #532] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_514357573_22 at /127.0.0.1:52152 [Waiting for operation #695] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=683 (was 605) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1988 (was 1669) - AvailableMemoryMB LEAK? - 2024-11-07T12:24:07,383 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=1987 2024-11-07T12:24:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:24:07,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:24:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:07,387 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:24:07,387 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:07,387 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-07T12:24:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:07,388 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:24:07,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742068_1244 (size=959) 2024-11-07T12:24:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:07,801 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:24:07,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742069_1245 (size=53) 2024-11-07T12:24:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:08,210 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:24:08,211 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5c03ab2185aa77a0f7f7a6bc80ad010c, disabling compactions & flushes 2024-11-07T12:24:08,211 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,211 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,211 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. after waiting 0 ms 2024-11-07T12:24:08,211 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,211 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,211 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: Waiting for close lock at 1730982248211Disabling compacts and flushes for region at 1730982248211Disabling writes for close at 1730982248211Writing region close event to WAL at 1730982248211Closed at 1730982248211 2024-11-07T12:24:08,213 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:24:08,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982248213"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982248213"}]},"ts":"1730982248213"} 2024-11-07T12:24:08,215 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:24:08,217 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:24:08,217 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982248217"}]},"ts":"1730982248217"} 2024-11-07T12:24:08,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:24:08,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, ASSIGN}] 2024-11-07T12:24:08,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, ASSIGN 2024-11-07T12:24:08,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:24:08,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5c03ab2185aa77a0f7f7a6bc80ad010c, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:08,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, ASSIGN because future has completed 2024-11-07T12:24:08,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:08,533 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,533 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:24:08,534 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,534 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:24:08,534 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,534 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,536 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,538 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c03ab2185aa77a0f7f7a6bc80ad010c columnFamilyName A 2024-11-07T12:24:08,538 DEBUG [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:08,539 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(327): Store=5c03ab2185aa77a0f7f7a6bc80ad010c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:08,539 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,540 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c03ab2185aa77a0f7f7a6bc80ad010c columnFamilyName B 2024-11-07T12:24:08,541 DEBUG [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:08,541 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(327): Store=5c03ab2185aa77a0f7f7a6bc80ad010c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:08,541 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,543 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c03ab2185aa77a0f7f7a6bc80ad010c columnFamilyName C 2024-11-07T12:24:08,543 DEBUG [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:08,544 INFO [StoreOpener-5c03ab2185aa77a0f7f7a6bc80ad010c-1 {}] regionserver.HStore(327): Store=5c03ab2185aa77a0f7f7a6bc80ad010c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:08,544 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,545 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,546 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,548 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,548 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,549 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:24:08,551 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,553 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:24:08,554 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 5c03ab2185aa77a0f7f7a6bc80ad010c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61475884, jitterRate=-0.08393794298171997}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:24:08,554 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:08,554 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: Running coprocessor pre-open hook at 1730982248534Writing region info on filesystem at 1730982248534Initializing all the Stores at 1730982248535 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982248535Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982248535Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982248536 (+1 ms)Cleaning up temporary data from old regions at 1730982248548 (+12 ms)Running coprocessor post-open hooks at 1730982248554 (+6 ms)Region opened successfully at 1730982248554 2024-11-07T12:24:08,555 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., pid=63, masterSystemTime=1730982248527 2024-11-07T12:24:08,557 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,557 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:08,557 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5c03ab2185aa77a0f7f7a6bc80ad010c, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:08,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:08,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T12:24:08,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 in 184 msec 2024-11-07T12:24:08,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T12:24:08,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, ASSIGN in 341 msec 2024-11-07T12:24:08,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:24:08,562 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982248562"}]},"ts":"1730982248562"} 2024-11-07T12:24:08,563 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:24:08,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:24:08,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-07T12:24:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T12:24:09,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:09,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79eacb52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,543 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515a8470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,544 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3903852d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,548 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:24:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:24:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7082e236 2024-11-07T12:24:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:24:09,549 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:24:09,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a054597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,551 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a846e89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8ae14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d1dc57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,553 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f88841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446dc9fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a525a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6db726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,557 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715cffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c2c082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,559 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7466d9b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a8b77d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29df300d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,562 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a151e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,563 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4102f281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c38859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5437c8a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9cbbc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3197c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@238ea3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da93613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533cbeb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,570 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162041d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b13b91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a14518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,574 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e35e8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669c8fe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef84e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:09,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:09,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4df60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:09,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:09,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2ce596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:09,580 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,580 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-07T12:24:09,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,583 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:09,584 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:24:09,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:09,588 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,588 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,589 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,589 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,590 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,591 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,591 DEBUG [Thread-1146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,591 DEBUG [Thread-1140 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,591 DEBUG [Thread-1146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,591 DEBUG [Thread-1140 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,592 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,592 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:09,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:09,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:24:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:24:09,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/108cbc1c198f4208bbd7c8e604ef1ee8 is 50, key is test_row_0/A:col10/1730982249587/Put/seqid=0 2024-11-07T12:24:09,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982309637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982309639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982309640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982309640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982309642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742070_1246 (size=12001) 2024-11-07T12:24:09,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/108cbc1c198f4208bbd7c8e604ef1ee8 2024-11-07T12:24:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:24:09,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/21bbd928637545d4acffc8b422f97718 is 50, key is test_row_0/B:col10/1730982249587/Put/seqid=0 2024-11-07T12:24:09,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742071_1247 (size=12001) 2024-11-07T12:24:09,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/21bbd928637545d4acffc8b422f97718 2024-11-07T12:24:09,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:24:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:09,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:09,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982309751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982309751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982309751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982309751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982309751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:09,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f9e4ff74e97c4c08aa06cdada1932337 is 50, key is test_row_0/C:col10/1730982249587/Put/seqid=0 2024-11-07T12:24:09,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742072_1248 (size=12001) 2024-11-07T12:24:09,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f9e4ff74e97c4c08aa06cdada1932337 2024-11-07T12:24:09,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/108cbc1c198f4208bbd7c8e604ef1ee8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8 2024-11-07T12:24:09,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:24:09,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/21bbd928637545d4acffc8b422f97718 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718 2024-11-07T12:24:09,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:24:09,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f9e4ff74e97c4c08aa06cdada1932337 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337 2024-11-07T12:24:09,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T12:24:09,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 230ms, sequenceid=12, compaction requested=false 2024-11-07T12:24:09,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:09,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T12:24:09,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:09,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:24:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:24:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/c92e6d77af414bef877382ab69d9997e is 50, key is test_row_0/A:col10/1730982249640/Put/seqid=0 2024-11-07T12:24:09,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742073_1249 (size=12001) 2024-11-07T12:24:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:10,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:24:10,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/c92e6d77af414bef877382ab69d9997e 2024-11-07T12:24:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c1d770461a8242999d76621df1abe1a6 is 50, key is test_row_0/B:col10/1730982249640/Put/seqid=0 2024-11-07T12:24:10,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742074_1250 (size=12001) 2024-11-07T12:24:10,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c1d770461a8242999d76621df1abe1a6 2024-11-07T12:24:10,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/98c60ef7226f46d3bf183d332183d135 is 50, key is test_row_0/C:col10/1730982249640/Put/seqid=0 2024-11-07T12:24:10,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742075_1251 (size=12001) 2024-11-07T12:24:10,420 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/98c60ef7226f46d3bf183d332183d135 2024-11-07T12:24:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/c92e6d77af414bef877382ab69d9997e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e 2024-11-07T12:24:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:24:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c1d770461a8242999d76621df1abe1a6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6 2024-11-07T12:24:10,442 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:24:10,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/98c60ef7226f46d3bf183d332183d135 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135 2024-11-07T12:24:10,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:24:10,453 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 561ms, sequenceid=37, compaction requested=false 2024-11-07T12:24:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-07T12:24:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-07T12:24:10,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T12:24:10,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 871 msec 2024-11-07T12:24:10,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 878 msec 2024-11-07T12:24:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:10,642 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:10,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:24:10,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/e028815ca14042bdaf206e842fa4f890 is 50, key is test_row_0/A:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:10,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742076_1252 (size=16681) 2024-11-07T12:24:10,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/e028815ca14042bdaf206e842fa4f890 2024-11-07T12:24:10,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/04714875b0a44f6d98d7e5631abd7915 is 50, key is test_row_0/B:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:10,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742077_1253 (size=12001) 2024-11-07T12:24:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T12:24:10,721 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:10,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T12:24:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:24:10,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:10,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:10,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:24:10,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:24:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:10,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:10,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982310981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982310981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982310981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982310981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982310981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:24:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:24:11,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/04714875b0a44f6d98d7e5631abd7915 2024-11-07T12:24:11,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2be015b9f4554f678715de8e5f0727d0 is 50, key is test_row_0/C:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:11,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742078_1254 (size=12001) 2024-11-07T12:24:11,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2be015b9f4554f678715de8e5f0727d0 2024-11-07T12:24:11,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/e028815ca14042bdaf206e842fa4f890 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890 2024-11-07T12:24:11,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890, entries=250, sequenceid=52, filesize=16.3 K 2024-11-07T12:24:11,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/04714875b0a44f6d98d7e5631abd7915 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915 2024-11-07T12:24:11,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T12:24:11,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2be015b9f4554f678715de8e5f0727d0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0 2024-11-07T12:24:11,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T12:24:11,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 496ms, sequenceid=52, compaction requested=true 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,138 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:11,138 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:11,139 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,139 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,139 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:11,139 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,139 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:11,139 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,139 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=35.2 K 2024-11-07T12:24:11,139 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=39.7 K 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bbd928637545d4acffc8b422f97718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982249587 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 108cbc1c198f4208bbd7c8e604ef1ee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982249587 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d770461a8242999d76621df1abe1a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982249633 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c92e6d77af414bef877382ab69d9997e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982249633 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e028815ca14042bdaf206e842fa4f890, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,140 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 04714875b0a44f6d98d7e5631abd7915, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,147 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,147 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/93f0740bcd9347afa0dbfc03ecf9527c is 50, key is test_row_0/A:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:11,149 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,149 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/77ee375092984475a93c83edb399ca63 is 50, key is test_row_0/B:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742079_1255 (size=12104) 2024-11-07T12:24:11,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742080_1256 (size=12104) 2024-11-07T12:24:11,167 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/93f0740bcd9347afa0dbfc03ecf9527c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/93f0740bcd9347afa0dbfc03ecf9527c 2024-11-07T12:24:11,173 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 93f0740bcd9347afa0dbfc03ecf9527c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:11,173 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,173 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=13, startTime=1730982251138; duration=0sec 2024-11-07T12:24:11,173 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:11,173 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:11,173 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,174 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/77ee375092984475a93c83edb399ca63 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/77ee375092984475a93c83edb399ca63 2024-11-07T12:24:11,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:11,174 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,174 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=35.2 K 2024-11-07T12:24:11,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e4ff74e97c4c08aa06cdada1932337, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730982249587 2024-11-07T12:24:11,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c60ef7226f46d3bf183d332183d135, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982249633 2024-11-07T12:24:11,176 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be015b9f4554f678715de8e5f0727d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,181 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into 77ee375092984475a93c83edb399ca63(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:11,181 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,181 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=13, startTime=1730982251138; duration=0sec 2024-11-07T12:24:11,181 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,181 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:11,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T12:24:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,183 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:24:11,183 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,184 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/a8fa570f95ad48dfa54d8efb8bc545fc is 50, key is test_row_0/C:col10/1730982250003/Put/seqid=0 2024-11-07T12:24:11,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62a46a3c9ab0439586537eb91616b6a0 is 50, key is test_row_0/A:col10/1730982250660/Put/seqid=0 2024-11-07T12:24:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742081_1257 (size=12104) 2024-11-07T12:24:11,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742082_1258 (size=12001) 2024-11-07T12:24:11,208 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62a46a3c9ab0439586537eb91616b6a0 2024-11-07T12:24:11,211 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/a8fa570f95ad48dfa54d8efb8bc545fc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/a8fa570f95ad48dfa54d8efb8bc545fc 2024-11-07T12:24:11,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/2eeefe8383674ebc8f8d8509bc291e41 is 50, key is test_row_0/B:col10/1730982250660/Put/seqid=0 2024-11-07T12:24:11,218 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into a8fa570f95ad48dfa54d8efb8bc545fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:11,218 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,219 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=13, startTime=1730982251138; duration=0sec 2024-11-07T12:24:11,219 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,219 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:11,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742083_1259 (size=12001) 2024-11-07T12:24:11,253 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/2eeefe8383674ebc8f8d8509bc291e41 2024-11-07T12:24:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/5017091b7e444691818dc5e253e5f0e3 is 50, key is test_row_0/C:col10/1730982250660/Put/seqid=0 2024-11-07T12:24:11,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742084_1260 (size=12001) 2024-11-07T12:24:11,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/5017091b7e444691818dc5e253e5f0e3 2024-11-07T12:24:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62a46a3c9ab0439586537eb91616b6a0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0 2024-11-07T12:24:11,279 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:24:11,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/2eeefe8383674ebc8f8d8509bc291e41 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41 2024-11-07T12:24:11,286 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:24:11,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/5017091b7e444691818dc5e253e5f0e3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3 2024-11-07T12:24:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:11,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:24:11,299 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 117ms, sequenceid=75, compaction requested=false 2024-11-07T12:24:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T12:24:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T12:24:11,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T12:24:11,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 578 msec 2024-11-07T12:24:11,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 583 msec 2024-11-07T12:24:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:11,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:24:11,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/b6c8bc777ba94364ac4f972e4cc6d174 is 50, key is test_row_0/A:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742085_1261 (size=16681) 2024-11-07T12:24:11,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/b6c8bc777ba94364ac4f972e4cc6d174 2024-11-07T12:24:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982311332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982311334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982311335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982311336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982311338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/4ac96dadcaf243b6bd30b78e24dafbf9 is 50, key is test_row_0/B:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T12:24:11,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:11,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T12:24:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:11,357 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:11,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:11,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:11,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742086_1262 (size=12001) 2024-11-07T12:24:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982311440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982311451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982311451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982311451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982311451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:11,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982311651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:11,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:11,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/4ac96dadcaf243b6bd30b78e24dafbf9 2024-11-07T12:24:11,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/394fc0df07834f3d968be6fa30a840aa is 50, key is test_row_0/C:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:11,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742087_1263 (size=12001) 2024-11-07T12:24:11,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/394fc0df07834f3d968be6fa30a840aa 2024-11-07T12:24:11,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/b6c8bc777ba94364ac4f972e4cc6d174 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174 2024-11-07T12:24:11,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174, entries=250, sequenceid=90, filesize=16.3 K 2024-11-07T12:24:11,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/4ac96dadcaf243b6bd30b78e24dafbf9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9 2024-11-07T12:24:11,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-07T12:24:11,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/394fc0df07834f3d968be6fa30a840aa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa 2024-11-07T12:24:11,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa, entries=150, sequenceid=90, filesize=11.7 K 2024-11-07T12:24:11,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 563ms, sequenceid=90, compaction requested=true 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,870 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:11,870 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:11,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:11,871 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,871 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:11,871 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,871 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/93f0740bcd9347afa0dbfc03ecf9527c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=39.8 K 2024-11-07T12:24:11,872 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,872 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:11,872 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,872 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/77ee375092984475a93c83edb399ca63, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=35.3 K 2024-11-07T12:24:11,872 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f0740bcd9347afa0dbfc03ecf9527c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,873 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a46a3c9ab0439586537eb91616b6a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982250659 2024-11-07T12:24:11,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ee375092984475a93c83edb399ca63, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,873 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c8bc777ba94364ac4f972e4cc6d174, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251295 2024-11-07T12:24:11,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eeefe8383674ebc8f8d8509bc291e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982250659 2024-11-07T12:24:11,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac96dadcaf243b6bd30b78e24dafbf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251304 2024-11-07T12:24:11,887 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,887 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,887 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/252221afcbb64b6ca1ba2a7cdfc934f7 is 50, key is test_row_0/B:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,887 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/bb2c69b0a81a42d5a4aead52088c5a42 is 50, key is test_row_0/A:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742088_1264 (size=12207) 2024-11-07T12:24:11,927 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/252221afcbb64b6ca1ba2a7cdfc934f7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/252221afcbb64b6ca1ba2a7cdfc934f7 2024-11-07T12:24:11,934 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into 252221afcbb64b6ca1ba2a7cdfc934f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:11,935 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:11,935 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=13, startTime=1730982251870; duration=0sec 2024-11-07T12:24:11,935 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:11,935 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:11,935 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:11,937 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:11,937 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:11,937 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,937 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/a8fa570f95ad48dfa54d8efb8bc545fc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=35.3 K 2024-11-07T12:24:11,939 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fa570f95ad48dfa54d8efb8bc545fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730982250003 2024-11-07T12:24:11,940 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5017091b7e444691818dc5e253e5f0e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982250659 2024-11-07T12:24:11,940 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394fc0df07834f3d968be6fa30a840aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251304 2024-11-07T12:24:11,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742089_1265 (size=12207) 2024-11-07T12:24:11,963 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:11,963 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:11,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:24:11,964 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/11db0afa0eb94f3f8f98978d6fc01bfa is 50, key is test_row_0/C:col10/1730982251304/Put/seqid=0 2024-11-07T12:24:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:11,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/76c233565d0c40ceb7583fc15b7b6b9f is 50, key is test_row_0/A:col10/1730982251336/Put/seqid=0 2024-11-07T12:24:11,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:11,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:11,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982311991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982311993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982311995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742090_1266 (size=12207) 2024-11-07T12:24:12,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982311998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982311999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,010 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/11db0afa0eb94f3f8f98978d6fc01bfa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11db0afa0eb94f3f8f98978d6fc01bfa 2024-11-07T12:24:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742091_1267 (size=14341) 2024-11-07T12:24:12,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/76c233565d0c40ceb7583fc15b7b6b9f 2024-11-07T12:24:12,020 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 11db0afa0eb94f3f8f98978d6fc01bfa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:12,021 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:12,021 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=13, startTime=1730982251870; duration=0sec 2024-11-07T12:24:12,021 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:12,021 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:12,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/409978e8104941cea73118fbd5e01943 is 50, key is test_row_0/B:col10/1730982251336/Put/seqid=0 2024-11-07T12:24:12,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742092_1268 (size=12001) 2024-11-07T12:24:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982312101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982312111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982312112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982312112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982312112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:12,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:12,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982312311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982312320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982312320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982312321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982312321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,358 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/bb2c69b0a81a42d5a4aead52088c5a42 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/bb2c69b0a81a42d5a4aead52088c5a42 2024-11-07T12:24:12,366 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into bb2c69b0a81a42d5a4aead52088c5a42(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:12,366 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:12,367 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=13, startTime=1730982251870; duration=0sec 2024-11-07T12:24:12,367 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:12,367 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:12,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:12,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/409978e8104941cea73118fbd5e01943 2024-11-07T12:24:12,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/22d61b5eec9448cf9903a2902c39d1b3 is 50, key is test_row_0/C:col10/1730982251336/Put/seqid=0 2024-11-07T12:24:12,485 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:24:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:12,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742093_1269 (size=12001) 2024-11-07T12:24:12,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/22d61b5eec9448cf9903a2902c39d1b3 2024-11-07T12:24:12,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/76c233565d0c40ceb7583fc15b7b6b9f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f 2024-11-07T12:24:12,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f, entries=200, sequenceid=114, filesize=14.0 K 2024-11-07T12:24:12,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/409978e8104941cea73118fbd5e01943 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943 2024-11-07T12:24:12,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T12:24:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/22d61b5eec9448cf9903a2902c39d1b3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3 2024-11-07T12:24:12,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T12:24:12,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 598ms, sequenceid=114, compaction requested=false 2024-11-07T12:24:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:12,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T12:24:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:12,589 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:24:12,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/056f69d739d547b8bbbb14c46b464cd4 is 50, key is test_row_0/A:col10/1730982251993/Put/seqid=0 2024-11-07T12:24:12,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742094_1270 (size=12001) 2024-11-07T12:24:12,615 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/056f69d739d547b8bbbb14c46b464cd4 2024-11-07T12:24:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/1fbab9ebbe8a4143ae55fc6aa12854cc is 50, key is test_row_0/B:col10/1730982251993/Put/seqid=0 2024-11-07T12:24:12,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742095_1271 (size=12001) 2024-11-07T12:24:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982312662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982312667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982312668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982312670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982312670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982312781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982312781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982312781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982312791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982312791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982312991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982312991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:12,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982312991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982313007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982313007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/1fbab9ebbe8a4143ae55fc6aa12854cc 2024-11-07T12:24:13,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f2670661386a4e23a44a05309f9fe0c3 is 50, key is test_row_0/C:col10/1730982251993/Put/seqid=0 2024-11-07T12:24:13,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742096_1272 (size=12001) 2024-11-07T12:24:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982313301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982313301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982313301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982313320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982313321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,491 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f2670661386a4e23a44a05309f9fe0c3 2024-11-07T12:24:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/056f69d739d547b8bbbb14c46b464cd4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4 2024-11-07T12:24:13,499 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4, entries=150, sequenceid=131, filesize=11.7 K 2024-11-07T12:24:13,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/1fbab9ebbe8a4143ae55fc6aa12854cc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc 2024-11-07T12:24:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:13,503 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc, entries=150, sequenceid=131, filesize=11.7 K 2024-11-07T12:24:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f2670661386a4e23a44a05309f9fe0c3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3 2024-11-07T12:24:13,508 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3, entries=150, sequenceid=131, filesize=11.7 K 2024-11-07T12:24:13,509 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 920ms, sequenceid=131, compaction requested=true 2024-11-07T12:24:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:13,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T12:24:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T12:24:13,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T12:24:13,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2024-11-07T12:24:13,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.1620 sec 2024-11-07T12:24:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:13,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:13,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:24:13,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/3151d97d144242179b795c192279ae22 is 50, key is test_row_0/A:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742097_1273 (size=12151) 2024-11-07T12:24:13,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982313828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982313831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982313831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982313832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982313832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982313941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982313941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:13,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982313950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982314151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982314151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982314161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/3151d97d144242179b795c192279ae22 2024-11-07T12:24:14,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/61cac69e453d4c0b966cb5ee0ba7778f is 50, key is test_row_0/B:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:14,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742098_1274 (size=12151) 2024-11-07T12:24:14,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/61cac69e453d4c0b966cb5ee0ba7778f 2024-11-07T12:24:14,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/4c2068f7fe444f34b0e9609867b18dd3 is 50, key is test_row_0/C:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:14,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742099_1275 (size=12151) 2024-11-07T12:24:14,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982314460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982314460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982314471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/4c2068f7fe444f34b0e9609867b18dd3 2024-11-07T12:24:14,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/3151d97d144242179b795c192279ae22 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22 2024-11-07T12:24:14,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:24:14,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/61cac69e453d4c0b966cb5ee0ba7778f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f 2024-11-07T12:24:14,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:24:14,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/4c2068f7fe444f34b0e9609867b18dd3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3 2024-11-07T12:24:14,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:24:14,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 878ms, sequenceid=155, compaction requested=true 2024-11-07T12:24:14,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:14,692 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:14,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:14,692 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:14,693 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:14,694 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:14,694 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:14,694 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/252221afcbb64b6ca1ba2a7cdfc934f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=47.2 K 2024-11-07T12:24:14,694 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 252221afcbb64b6ca1ba2a7cdfc934f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251304 2024-11-07T12:24:14,695 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:14,695 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:14,695 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:14,695 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/bb2c69b0a81a42d5a4aead52088c5a42, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=49.5 K 2024-11-07T12:24:14,695 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 409978e8104941cea73118fbd5e01943, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982251333 2024-11-07T12:24:14,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2c69b0a81a42d5a4aead52088c5a42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251304 2024-11-07T12:24:14,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbab9ebbe8a4143ae55fc6aa12854cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982251991 2024-11-07T12:24:14,696 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c233565d0c40ceb7583fc15b7b6b9f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982251333 2024-11-07T12:24:14,696 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 61cac69e453d4c0b966cb5ee0ba7778f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:14,697 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 056f69d739d547b8bbbb14c46b464cd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982251991 2024-11-07T12:24:14,698 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3151d97d144242179b795c192279ae22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:14,711 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:14,712 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/99d0caeec36e4272aafa2df46eb19493 is 50, key is test_row_0/A:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:14,719 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:14,720 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/8024f3ee85614cf6a6c029ff449ac975 is 50, key is test_row_0/B:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:14,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742100_1276 (size=12493) 2024-11-07T12:24:14,745 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/99d0caeec36e4272aafa2df46eb19493 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/99d0caeec36e4272aafa2df46eb19493 2024-11-07T12:24:14,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742101_1277 (size=12493) 2024-11-07T12:24:14,752 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 99d0caeec36e4272aafa2df46eb19493(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:14,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:14,752 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=12, startTime=1730982254692; duration=0sec 2024-11-07T12:24:14,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:14,752 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:14,753 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:14,755 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:14,755 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:14,755 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:14,755 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11db0afa0eb94f3f8f98978d6fc01bfa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=47.2 K 2024-11-07T12:24:14,755 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11db0afa0eb94f3f8f98978d6fc01bfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1730982251304 2024-11-07T12:24:14,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d61b5eec9448cf9903a2902c39d1b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982251333 2024-11-07T12:24:14,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2670661386a4e23a44a05309f9fe0c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982251991 2024-11-07T12:24:14,757 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2068f7fe444f34b0e9609867b18dd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:14,781 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:14,782 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/06776dc3bb814359b673d12db49f568f is 50, key is test_row_0/C:col10/1730982252667/Put/seqid=0 2024-11-07T12:24:14,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742102_1278 (size=12493) 2024-11-07T12:24:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:14,843 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:14,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:24:14,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/30f08371d9d54639ac71a071700b69e9 is 50, key is test_row_0/A:col10/1730982254842/Put/seqid=0 2024-11-07T12:24:14,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742103_1279 (size=14541) 2024-11-07T12:24:14,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982314893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982314897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982314971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982314971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982314981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982315001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982315010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,158 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/8024f3ee85614cf6a6c029ff449ac975 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8024f3ee85614cf6a6c029ff449ac975 2024-11-07T12:24:15,170 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into 8024f3ee85614cf6a6c029ff449ac975(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:15,170 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:15,170 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=12, startTime=1730982254692; duration=0sec 2024-11-07T12:24:15,171 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:15,171 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:15,205 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/06776dc3bb814359b673d12db49f568f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/06776dc3bb814359b673d12db49f568f 2024-11-07T12:24:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982315211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,213 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 06776dc3bb814359b673d12db49f568f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:15,213 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:15,213 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=12, startTime=1730982254692; duration=0sec 2024-11-07T12:24:15,213 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:15,213 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982315221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/30f08371d9d54639ac71a071700b69e9 2024-11-07T12:24:15,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/8a2941f1b01349f5b5fdf855b2e05804 is 50, key is test_row_0/B:col10/1730982254842/Put/seqid=0 2024-11-07T12:24:15,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742104_1280 (size=12151) 2024-11-07T12:24:15,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/8a2941f1b01349f5b5fdf855b2e05804 2024-11-07T12:24:15,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f81103be32254c8ca123e641354b359b is 50, key is test_row_0/C:col10/1730982254842/Put/seqid=0 2024-11-07T12:24:15,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742105_1281 (size=12151) 2024-11-07T12:24:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T12:24:15,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982315521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T12:24:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:15,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:15,524 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:15,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982315531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:15,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:24:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:15,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:15,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f81103be32254c8ca123e641354b359b 2024-11-07T12:24:15,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/30f08371d9d54639ac71a071700b69e9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9 2024-11-07T12:24:15,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9, entries=200, sequenceid=170, filesize=14.2 K 2024-11-07T12:24:15,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/8a2941f1b01349f5b5fdf855b2e05804 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804 2024-11-07T12:24:15,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T12:24:15,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f81103be32254c8ca123e641354b359b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b 2024-11-07T12:24:15,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T12:24:15,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 910ms, sequenceid=170, compaction requested=false 2024-11-07T12:24:15,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:15,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T12:24:15,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:15,841 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:24:15,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/232e663e412a49d6970271dc25d67912 is 50, key is test_row_0/A:col10/1730982254888/Put/seqid=0 2024-11-07T12:24:15,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742106_1282 (size=12151) 2024-11-07T12:24:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:15,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:16,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982316001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982316002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982316003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982316031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982316041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:16,291 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/232e663e412a49d6970271dc25d67912 2024-11-07T12:24:16,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e58fe55590e14041a028b92e19af6a60 is 50, key is test_row_0/B:col10/1730982254888/Put/seqid=0 2024-11-07T12:24:16,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742107_1283 (size=12151) 2024-11-07T12:24:16,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e58fe55590e14041a028b92e19af6a60 2024-11-07T12:24:16,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/c558127d70934c4fad24b9f034d41240 is 50, key is test_row_0/C:col10/1730982254888/Put/seqid=0 2024-11-07T12:24:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982316322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982316324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982316324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742108_1284 (size=12151) 2024-11-07T12:24:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982316630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982316631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982316641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:16,734 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/c558127d70934c4fad24b9f034d41240 2024-11-07T12:24:16,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/232e663e412a49d6970271dc25d67912 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912 2024-11-07T12:24:16,745 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:24:16,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e58fe55590e14041a028b92e19af6a60 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60 2024-11-07T12:24:16,752 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:24:16,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/c558127d70934c4fad24b9f034d41240 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240 2024-11-07T12:24:16,762 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240, entries=150, sequenceid=194, filesize=11.9 K 2024-11-07T12:24:16,763 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 922ms, sequenceid=194, compaction requested=true 2024-11-07T12:24:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T12:24:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T12:24:16,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T12:24:16,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2420 sec 2024-11-07T12:24:16,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2500 sec 2024-11-07T12:24:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:17,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:24:17,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/11ebbb7834854ea39cf14c63cedbf5a2 is 50, key is test_row_0/A:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:17,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742109_1285 (size=14541) 2024-11-07T12:24:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982317099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982317102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982317141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982317141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982317150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982317211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982317211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982317420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982317420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/11ebbb7834854ea39cf14c63cedbf5a2 2024-11-07T12:24:17,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e86ef91851194c1d803c3da4eb09a197 is 50, key is test_row_0/B:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:17,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742110_1286 (size=12151) 2024-11-07T12:24:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T12:24:17,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:17,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T12:24:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:17,672 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:17,673 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:17,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982317731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982317731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:17,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:24:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:17,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e86ef91851194c1d803c3da4eb09a197 2024-11-07T12:24:17,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/6815c6ed8de045c0a01d2f232b9e50e7 is 50, key is test_row_0/C:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:17,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742111_1287 (size=12151) 2024-11-07T12:24:17,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:24:17,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:17,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:17,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:17,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:18,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:24:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:18,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982318151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982318151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:18,154 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:18,154 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:18,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982318160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:18,162 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982318241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982318241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:18,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:24:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:18,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/6815c6ed8de045c0a01d2f232b9e50e7 2024-11-07T12:24:18,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/11ebbb7834854ea39cf14c63cedbf5a2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2 2024-11-07T12:24:18,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2, entries=200, sequenceid=209, filesize=14.2 K 2024-11-07T12:24:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:18,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e86ef91851194c1d803c3da4eb09a197 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197 2024-11-07T12:24:18,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:24:18,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/6815c6ed8de045c0a01d2f232b9e50e7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7 2024-11-07T12:24:18,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T12:24:18,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1255ms, sequenceid=209, compaction requested=true 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:18,310 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:18,310 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:18,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:18,313 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,313 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,313 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/99d0caeec36e4272aafa2df46eb19493, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=52.5 K 2024-11-07T12:24:18,313 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8024f3ee85614cf6a6c029ff449ac975, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=47.8 K 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d0caeec36e4272aafa2df46eb19493, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:18,313 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 8024f3ee85614cf6a6c029ff449ac975, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f08371d9d54639ac71a071700b69e9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730982253826 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2941f1b01349f5b5fdf855b2e05804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730982253826 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e58fe55590e14041a028b92e19af6a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730982254887 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232e663e412a49d6970271dc25d67912, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730982254887 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e86ef91851194c1d803c3da4eb09a197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:18,314 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ebbb7834854ea39cf14c63cedbf5a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:18,322 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:18,322 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:18,322 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/554bd880bf4a44c786888f1a3d5f8f85 is 50, key is test_row_0/A:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:18,325 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/da3ffde3c15243b09eef6ab2b162ee18 is 50, key is test_row_0/B:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:18,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742112_1288 (size=12629) 2024-11-07T12:24:18,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742113_1289 (size=12629) 2024-11-07T12:24:18,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T12:24:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,436 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:24:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/00db73e8d8114ccebb34c7280bd2094a is 50, key is test_row_0/A:col10/1730982257101/Put/seqid=0 2024-11-07T12:24:18,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742114_1290 (size=12151) 2024-11-07T12:24:18,749 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/554bd880bf4a44c786888f1a3d5f8f85 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/554bd880bf4a44c786888f1a3d5f8f85 2024-11-07T12:24:18,752 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/da3ffde3c15243b09eef6ab2b162ee18 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/da3ffde3c15243b09eef6ab2b162ee18 2024-11-07T12:24:18,755 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 554bd880bf4a44c786888f1a3d5f8f85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:18,755 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:18,756 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=12, startTime=1730982258310; duration=0sec 2024-11-07T12:24:18,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:18,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:18,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:18,758 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into da3ffde3c15243b09eef6ab2b162ee18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:18,758 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:18,758 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=12, startTime=1730982258310; duration=0sec 2024-11-07T12:24:18,758 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:18,758 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:18,758 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:18,758 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:18,758 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:18,758 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/06776dc3bb814359b673d12db49f568f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=47.8 K 2024-11-07T12:24:18,760 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06776dc3bb814359b673d12db49f568f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982252667 2024-11-07T12:24:18,760 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81103be32254c8ca123e641354b359b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730982253826 2024-11-07T12:24:18,761 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c558127d70934c4fad24b9f034d41240, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1730982254887 2024-11-07T12:24:18,761 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6815c6ed8de045c0a01d2f232b9e50e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:18,772 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:18,773 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/11561f27a9c140c79c83194e45fc29df is 50, key is test_row_0/C:col10/1730982255996/Put/seqid=0 2024-11-07T12:24:18,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742115_1291 (size=12629) 2024-11-07T12:24:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:18,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/00db73e8d8114ccebb34c7280bd2094a 2024-11-07T12:24:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/43477082b9b64e429d0620ea46d05890 is 50, key is test_row_0/B:col10/1730982257101/Put/seqid=0 2024-11-07T12:24:18,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742116_1292 (size=12151) 2024-11-07T12:24:19,187 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/11561f27a9c140c79c83194e45fc29df as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11561f27a9c140c79c83194e45fc29df 2024-11-07T12:24:19,192 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 11561f27a9c140c79c83194e45fc29df(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:19,192 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:19,193 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=12, startTime=1730982258310; duration=0sec 2024-11-07T12:24:19,193 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:19,193 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:19,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:19,274 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/43477082b9b64e429d0620ea46d05890 2024-11-07T12:24:19,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f350c78cd9034b99835a09b28a22d0da is 50, key is test_row_0/C:col10/1730982257101/Put/seqid=0 2024-11-07T12:24:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982319282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982319284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742117_1293 (size=12151) 2024-11-07T12:24:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982319390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982319390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982319601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982319601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,691 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f350c78cd9034b99835a09b28a22d0da 2024-11-07T12:24:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/00db73e8d8114ccebb34c7280bd2094a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a 2024-11-07T12:24:19,698 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T12:24:19,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/43477082b9b64e429d0620ea46d05890 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890 2024-11-07T12:24:19,703 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T12:24:19,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/f350c78cd9034b99835a09b28a22d0da as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da 2024-11-07T12:24:19,708 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T12:24:19,709 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1273ms, sequenceid=230, compaction requested=false 2024-11-07T12:24:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T12:24:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T12:24:19,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T12:24:19,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-07T12:24:19,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0420 sec 2024-11-07T12:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T12:24:19,821 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:19,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T12:24:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:19,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:19,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:19,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:19,913 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:19,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:24:19,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/015683a06f564ffeac14ef95557dd973 is 50, key is test_row_0/A:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:19,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742118_1294 (size=14541) 2024-11-07T12:24:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:19,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982319943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982319945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:19,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:19,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:19,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982320051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982320051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:20,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982320181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982320181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,185 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,185 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982320190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,193 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982320261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982320261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/015683a06f564ffeac14ef95557dd973 2024-11-07T12:24:20,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e3098bc8c187423a91076e669d09ad0a is 50, key is test_row_0/B:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:20,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742119_1295 (size=12151) 2024-11-07T12:24:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:20,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982320570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982320570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:20,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e3098bc8c187423a91076e669d09ad0a 2024-11-07T12:24:20,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ecb5db1ecc8c41969379df5c07c93b40 is 50, key is test_row_0/C:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742120_1296 (size=12151) 2024-11-07T12:24:20,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:20,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982321080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982321080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:21,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:21,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ecb5db1ecc8c41969379df5c07c93b40 2024-11-07T12:24:21,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/015683a06f564ffeac14ef95557dd973 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973 2024-11-07T12:24:21,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973, entries=200, sequenceid=250, filesize=14.2 K 2024-11-07T12:24:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e3098bc8c187423a91076e669d09ad0a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a 2024-11-07T12:24:21,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T12:24:21,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ecb5db1ecc8c41969379df5c07c93b40 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40 2024-11-07T12:24:21,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T12:24:21,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1246ms, sequenceid=250, compaction requested=true 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:21,159 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:21,159 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:21,160 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:21,160 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:21,160 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:21,160 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:21,160 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,160 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,160 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/554bd880bf4a44c786888f1a3d5f8f85, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=38.4 K 2024-11-07T12:24:21,160 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/da3ffde3c15243b09eef6ab2b162ee18, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=36.1 K 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting da3ffde3c15243b09eef6ab2b162ee18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554bd880bf4a44c786888f1a3d5f8f85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 43477082b9b64e429d0620ea46d05890, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730982257094 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00db73e8d8114ccebb34c7280bd2094a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730982257094 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e3098bc8c187423a91076e669d09ad0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259283 2024-11-07T12:24:21,161 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015683a06f564ffeac14ef95557dd973, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259273 2024-11-07T12:24:21,167 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:21,167 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/5a32851448784636abc1d8d0c9ccaee6 is 50, key is test_row_0/A:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:21,170 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:21,170 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/3ce0f463c3444726b26b3214144c159d is 50, key is test_row_0/B:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742121_1297 (size=12731) 2024-11-07T12:24:21,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742122_1298 (size=12731) 2024-11-07T12:24:21,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T12:24:21,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,238 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:24:21,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/6d5a4f5fdcc542f4b3aa0339cd729400 is 50, key is test_row_0/A:col10/1730982259944/Put/seqid=0 2024-11-07T12:24:21,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742123_1299 (size=12301) 2024-11-07T12:24:21,577 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/3ce0f463c3444726b26b3214144c159d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/3ce0f463c3444726b26b3214144c159d 2024-11-07T12:24:21,577 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/5a32851448784636abc1d8d0c9ccaee6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5a32851448784636abc1d8d0c9ccaee6 2024-11-07T12:24:21,581 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into 3ce0f463c3444726b26b3214144c159d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:21,581 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 5a32851448784636abc1d8d0c9ccaee6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:21,581 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=13, startTime=1730982261159; duration=0sec 2024-11-07T12:24:21,581 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=13, startTime=1730982261159; duration=0sec 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:21,581 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:21,582 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:21,583 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:21,583 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:21,583 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:21,583 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11561f27a9c140c79c83194e45fc29df, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=36.1 K 2024-11-07T12:24:21,583 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11561f27a9c140c79c83194e45fc29df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730982255996 2024-11-07T12:24:21,583 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f350c78cd9034b99835a09b28a22d0da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730982257094 2024-11-07T12:24:21,584 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb5db1ecc8c41969379df5c07c93b40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259283 2024-11-07T12:24:21,591 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:21,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/58a088004833462b9132857f1b37e08b is 50, key is test_row_0/C:col10/1730982259911/Put/seqid=0 2024-11-07T12:24:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742124_1300 (size=12731) 2024-11-07T12:24:21,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/6d5a4f5fdcc542f4b3aa0339cd729400 2024-11-07T12:24:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/caed363642ff420daacce1c4c450d4e9 is 50, key is test_row_0/B:col10/1730982259944/Put/seqid=0 2024-11-07T12:24:21,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742125_1301 (size=12301) 2024-11-07T12:24:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:21,998 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/58a088004833462b9132857f1b37e08b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/58a088004833462b9132857f1b37e08b 2024-11-07T12:24:22,002 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 58a088004833462b9132857f1b37e08b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:22,002 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:22,002 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=13, startTime=1730982261159; duration=0sec 2024-11-07T12:24:22,003 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:22,003 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:22,071 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/caed363642ff420daacce1c4c450d4e9 2024-11-07T12:24:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/b96b3337c1f34499b748b72efad61df9 is 50, key is test_row_0/C:col10/1730982259944/Put/seqid=0 2024-11-07T12:24:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742126_1302 (size=12301) 2024-11-07T12:24:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:22,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:22,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982322137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982322141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982322251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982322251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982322461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982322461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/b96b3337c1f34499b748b72efad61df9 2024-11-07T12:24:22,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/6d5a4f5fdcc542f4b3aa0339cd729400 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400 2024-11-07T12:24:22,489 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400, entries=150, sequenceid=269, filesize=12.0 K 2024-11-07T12:24:22,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/caed363642ff420daacce1c4c450d4e9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9 2024-11-07T12:24:22,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9, entries=150, sequenceid=269, filesize=12.0 K 2024-11-07T12:24:22,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/b96b3337c1f34499b748b72efad61df9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9 2024-11-07T12:24:22,498 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9, entries=150, sequenceid=269, filesize=12.0 K 2024-11-07T12:24:22,499 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1261ms, sequenceid=269, compaction requested=false 2024-11-07T12:24:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T12:24:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T12:24:22,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T12:24:22,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6780 sec 2024-11-07T12:24:22,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.6820 sec 2024-11-07T12:24:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:22,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:22,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:24:22,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/82e2bb2529124132886184187f0c77e1 is 50, key is test_row_0/A:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:22,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742127_1303 (size=14741) 2024-11-07T12:24:22,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/82e2bb2529124132886184187f0c77e1 2024-11-07T12:24:22,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/003453078c5f4ae6ac0bb7f62830d17b is 50, key is test_row_0/B:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742128_1304 (size=12301) 2024-11-07T12:24:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982322803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982322804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982322911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982322911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982323120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982323120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/003453078c5f4ae6ac0bb7f62830d17b 2024-11-07T12:24:23,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/df31869c79e94d779c9856f5ac307957 is 50, key is test_row_0/C:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:23,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742129_1305 (size=12301) 2024-11-07T12:24:23,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982323431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982323431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/df31869c79e94d779c9856f5ac307957 2024-11-07T12:24:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/82e2bb2529124132886184187f0c77e1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1 2024-11-07T12:24:23,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1, entries=200, sequenceid=291, filesize=14.4 K 2024-11-07T12:24:23,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/003453078c5f4ae6ac0bb7f62830d17b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b 2024-11-07T12:24:23,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-07T12:24:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/df31869c79e94d779c9856f5ac307957 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957 2024-11-07T12:24:23,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957, entries=150, sequenceid=291, filesize=12.0 K 2024-11-07T12:24:23,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 846ms, sequenceid=291, compaction requested=true 2024-11-07T12:24:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:23,617 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:23,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:23,618 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:23,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:23,618 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:23,618 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:23,618 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:23,618 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:23,619 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5a32851448784636abc1d8d0c9ccaee6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=38.8 K 2024-11-07T12:24:23,619 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/3ce0f463c3444726b26b3214144c159d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=36.5 K 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a32851448784636abc1d8d0c9ccaee6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259283 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce0f463c3444726b26b3214144c159d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259283 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting caed363642ff420daacce1c4c450d4e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1730982259940 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5a4f5fdcc542f4b3aa0339cd729400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1730982259940 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e2bb2529124132886184187f0c77e1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:23,619 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 003453078c5f4ae6ac0bb7f62830d17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:23,628 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:23,629 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:23,629 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/ecb2405fd8374d72bdb1e8b9b49ddd57 is 50, key is test_row_0/B:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:23,629 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/36830c0f74c14c2f945fc1bfb0efa803 is 50, key is test_row_0/A:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:23,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742130_1306 (size=12983) 2024-11-07T12:24:23,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742131_1307 (size=12983) 2024-11-07T12:24:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:23,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:23,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:24:23,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/749c8862386748f1950129e82fadf47c is 50, key is test_row_0/A:col10/1730982263942/Put/seqid=0 2024-11-07T12:24:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742132_1308 (size=12301) 2024-11-07T12:24:23,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982323975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982323976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T12:24:23,981 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:23,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T12:24:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:23,983 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:23,983 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:23,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:24,046 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/36830c0f74c14c2f945fc1bfb0efa803 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/36830c0f74c14c2f945fc1bfb0efa803 2024-11-07T12:24:24,050 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/ecb2405fd8374d72bdb1e8b9b49ddd57 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/ecb2405fd8374d72bdb1e8b9b49ddd57 2024-11-07T12:24:24,050 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 36830c0f74c14c2f945fc1bfb0efa803(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:24,050 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:24,050 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=13, startTime=1730982263617; duration=0sec 2024-11-07T12:24:24,050 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:24,051 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:24,051 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:24,051 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:24,052 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:24,052 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,052 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/58a088004833462b9132857f1b37e08b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=36.5 K 2024-11-07T12:24:24,052 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a088004833462b9132857f1b37e08b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982259283 2024-11-07T12:24:24,052 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b96b3337c1f34499b748b72efad61df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1730982259940 2024-11-07T12:24:24,053 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting df31869c79e94d779c9856f5ac307957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:24,054 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into ecb2405fd8374d72bdb1e8b9b49ddd57(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:24,054 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:24,054 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=13, startTime=1730982263617; duration=0sec 2024-11-07T12:24:24,055 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:24,055 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:24,060 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:24,061 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/37791aed1fbd49238b91d19a0902fe0d is 50, key is test_row_0/C:col10/1730982262772/Put/seqid=0 2024-11-07T12:24:24,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742133_1309 (size=12983) 2024-11-07T12:24:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982324081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982324090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40476 deadline: 1730982324190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,193 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40452 deadline: 1730982324201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40416 deadline: 1730982324201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,203 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,203 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in 5c03ab2185aa77a0f7f7a6bc80ad010c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982324290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982324301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/749c8862386748f1950129e82fadf47c 2024-11-07T12:24:24,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e013e62a013a409996d75e979a87a470 is 50, key is test_row_0/B:col10/1730982263942/Put/seqid=0 2024-11-07T12:24:24,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742134_1310 (size=12301) 2024-11-07T12:24:24,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,469 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/37791aed1fbd49238b91d19a0902fe0d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/37791aed1fbd49238b91d19a0902fe0d 2024-11-07T12:24:24,473 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 37791aed1fbd49238b91d19a0902fe0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:24,473 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:24,473 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=13, startTime=1730982263618; duration=0sec 2024-11-07T12:24:24,473 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:24,473 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:24,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982324601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982324610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e013e62a013a409996d75e979a87a470 2024-11-07T12:24:24,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2bf471f8c3c1414da641dd8786a8157f is 50, key is test_row_0/C:col10/1730982263942/Put/seqid=0 2024-11-07T12:24:24,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742135_1311 (size=12301) 2024-11-07T12:24:24,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:24,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:25,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:25,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:25,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982325111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:25,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982325120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:25,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2bf471f8c3c1414da641dd8786a8157f 2024-11-07T12:24:25,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/749c8862386748f1950129e82fadf47c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c 2024-11-07T12:24:25,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T12:24:25,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e013e62a013a409996d75e979a87a470 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470 2024-11-07T12:24:25,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T12:24:25,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/2bf471f8c3c1414da641dd8786a8157f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f 2024-11-07T12:24:25,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T12:24:25,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1245ms, sequenceid=309, compaction requested=false 2024-11-07T12:24:25,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:25,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T12:24:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:25,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:24:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/2bc45b7a596d4290a666c92ae7503107 is 50, key is test_row_0/A:col10/1730982263975/Put/seqid=0 2024-11-07T12:24:25,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742136_1312 (size=12301) 2024-11-07T12:24:25,615 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/2bc45b7a596d4290a666c92ae7503107 2024-11-07T12:24:25,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/aad6b41ee3224d688d0298f4bb7c05ff is 50, key is test_row_0/B:col10/1730982263975/Put/seqid=0 2024-11-07T12:24:25,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742137_1313 (size=12301) 2024-11-07T12:24:26,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/aad6b41ee3224d688d0298f4bb7c05ff 2024-11-07T12:24:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/bf16e82fb54e4325811d7a4c08ee1df0 is 50, key is test_row_0/C:col10/1730982263975/Put/seqid=0 2024-11-07T12:24:26,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742138_1314 (size=12301) 2024-11-07T12:24:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:26,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:26,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982326158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982326158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982326271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982326271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,439 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/bf16e82fb54e4325811d7a4c08ee1df0 2024-11-07T12:24:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/2bc45b7a596d4290a666c92ae7503107 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107 2024-11-07T12:24:26,446 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T12:24:26,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/aad6b41ee3224d688d0298f4bb7c05ff as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff 2024-11-07T12:24:26,450 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T12:24:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/bf16e82fb54e4325811d7a4c08ee1df0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0 2024-11-07T12:24:26,454 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T12:24:26,454 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1248ms, sequenceid=330, compaction requested=true 2024-11-07T12:24:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T12:24:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T12:24:26,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T12:24:26,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-07T12:24:26,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.4770 sec 2024-11-07T12:24:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:26,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:24:26,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/1faf55b29e794704a7305f1f7253c775 is 50, key is test_row_0/A:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742139_1315 (size=14741) 2024-11-07T12:24:26,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982326514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982326516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982326621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982326631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982326830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982326841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:26,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/1faf55b29e794704a7305f1f7253c775 2024-11-07T12:24:26,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e46f4fe11bad412599361a77a1407c9f is 50, key is test_row_0/B:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:26,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742140_1316 (size=12301) 2024-11-07T12:24:26,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e46f4fe11bad412599361a77a1407c9f 2024-11-07T12:24:26,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/e4f7ace8870e4d3989ff333ea1ff1aec is 50, key is test_row_0/C:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:26,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742141_1317 (size=12301) 2024-11-07T12:24:26,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/e4f7ace8870e4d3989ff333ea1ff1aec 2024-11-07T12:24:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/1faf55b29e794704a7305f1f7253c775 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775 2024-11-07T12:24:26,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775, entries=200, sequenceid=351, filesize=14.4 K 2024-11-07T12:24:26,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/e46f4fe11bad412599361a77a1407c9f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f 2024-11-07T12:24:26,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:24:26,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/e4f7ace8870e4d3989ff333ea1ff1aec as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec 2024-11-07T12:24:26,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec, entries=150, sequenceid=351, filesize=12.0 K 2024-11-07T12:24:26,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 456ms, sequenceid=351, compaction requested=true 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:26,939 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:24:26,939 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:26,941 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:26,941 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:26,941 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:26,941 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/36830c0f74c14c2f945fc1bfb0efa803, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=51.1 K 2024-11-07T12:24:26,941 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36830c0f74c14c2f945fc1bfb0efa803, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:26,942 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:26,942 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/ecb2405fd8374d72bdb1e8b9b49ddd57, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=48.7 K 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749c8862386748f1950129e82fadf47c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730982262798 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc45b7a596d4290a666c92ae7503107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730982263970 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ecb2405fd8374d72bdb1e8b9b49ddd57, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e013e62a013a409996d75e979a87a470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730982262798 2024-11-07T12:24:26,942 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1faf55b29e794704a7305f1f7253c775, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:26,943 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting aad6b41ee3224d688d0298f4bb7c05ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730982263970 2024-11-07T12:24:26,943 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e46f4fe11bad412599361a77a1407c9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:26,953 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#267 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:26,954 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83 is 50, key is test_row_0/B:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:26,964 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#268 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:26,964 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/3b52ba1c836b4a27ab098fcfe049cc96 is 50, key is test_row_0/A:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:26,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742142_1318 (size=13119) 2024-11-07T12:24:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742143_1319 (size=13119) 2024-11-07T12:24:26,977 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83 2024-11-07T12:24:26,986 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into f4ddcdd1e9bf4b27b3f2c3be49d27a83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:26,986 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:26,986 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=12, startTime=1730982266939; duration=0sec 2024-11-07T12:24:26,986 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:26,986 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:26,986 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:26,987 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:26,987 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:26,987 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:26,987 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/37791aed1fbd49238b91d19a0902fe0d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=48.7 K 2024-11-07T12:24:26,988 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 37791aed1fbd49238b91d19a0902fe0d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1730982262135 2024-11-07T12:24:26,989 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf471f8c3c1414da641dd8786a8157f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1730982262798 2024-11-07T12:24:26,989 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bf16e82fb54e4325811d7a4c08ee1df0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730982263970 2024-11-07T12:24:26,990 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f7ace8870e4d3989ff333ea1ff1aec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:27,000 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#269 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:27,000 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/47f1e5afdbc945569d386daa74f68f1a is 50, key is test_row_0/C:col10/1730982266157/Put/seqid=0 2024-11-07T12:24:27,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742144_1320 (size=13119) 2024-11-07T12:24:27,017 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/47f1e5afdbc945569d386daa74f68f1a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/47f1e5afdbc945569d386daa74f68f1a 2024-11-07T12:24:27,023 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 47f1e5afdbc945569d386daa74f68f1a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:27,023 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:27,023 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=12, startTime=1730982266939; duration=0sec 2024-11-07T12:24:27,023 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:27,023 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:27,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:24:27,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/a54ea20aafba439bbd4a854354a5b9b3 is 50, key is test_row_0/A:col10/1730982266512/Put/seqid=0 2024-11-07T12:24:27,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742145_1321 (size=14741) 2024-11-07T12:24:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982327180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982327183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982327291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982327291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,377 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/3b52ba1c836b4a27ab098fcfe049cc96 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3b52ba1c836b4a27ab098fcfe049cc96 2024-11-07T12:24:27,385 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 3b52ba1c836b4a27ab098fcfe049cc96(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:27,385 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:27,385 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=12, startTime=1730982266939; duration=0sec 2024-11-07T12:24:27,385 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:27,385 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982327500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982327501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/a54ea20aafba439bbd4a854354a5b9b3 2024-11-07T12:24:27,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c11a74df6cb54d0a818efaff07c38fe3 is 50, key is test_row_0/B:col10/1730982266512/Put/seqid=0 2024-11-07T12:24:27,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742146_1322 (size=12301) 2024-11-07T12:24:27,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c11a74df6cb54d0a818efaff07c38fe3 2024-11-07T12:24:27,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ad4c88a08d2b401f9362840ca9bb7537 is 50, key is test_row_0/C:col10/1730982266512/Put/seqid=0 2024-11-07T12:24:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742147_1323 (size=12301) 2024-11-07T12:24:27,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982327811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982327811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:27,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:27,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ad4c88a08d2b401f9362840ca9bb7537 2024-11-07T12:24:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/a54ea20aafba439bbd4a854354a5b9b3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3 2024-11-07T12:24:27,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3, entries=200, sequenceid=369, filesize=14.4 K 2024-11-07T12:24:27,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/c11a74df6cb54d0a818efaff07c38fe3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3 2024-11-07T12:24:27,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T12:24:27,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/ad4c88a08d2b401f9362840ca9bb7537 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537 2024-11-07T12:24:28,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T12:24:28,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 861ms, sequenceid=369, compaction requested=false 2024-11-07T12:24:28,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T12:24:28,141 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:28,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T12:24:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:24:28,143 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:28,144 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:28,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:24:28,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T12:24:28,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:28,296 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:24:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/5ef7b2fd3e304ef4a0256f3cc7622f76 is 50, key is test_row_0/A:col10/1730982267179/Put/seqid=0 2024-11-07T12:24:28,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742148_1324 (size=12301) 2024-11-07T12:24:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:28,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982328341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982328345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982328451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982328451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:24:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982328660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982328661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/5ef7b2fd3e304ef4a0256f3cc7622f76 2024-11-07T12:24:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/682178f8ee0c420386afd64a69add125 is 50, key is test_row_0/B:col10/1730982267179/Put/seqid=0 2024-11-07T12:24:28,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742149_1325 (size=12301) 2024-11-07T12:24:28,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/682178f8ee0c420386afd64a69add125 2024-11-07T12:24:28,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/265b2e263eee45d98c97321bdf6b16bd is 50, key is test_row_0/C:col10/1730982267179/Put/seqid=0 2024-11-07T12:24:28,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742150_1326 (size=12301) 2024-11-07T12:24:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:24:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982328971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982328971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,128 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/265b2e263eee45d98c97321bdf6b16bd 2024-11-07T12:24:29,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/5ef7b2fd3e304ef4a0256f3cc7622f76 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76 2024-11-07T12:24:29,134 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76, entries=150, sequenceid=390, filesize=12.0 K 2024-11-07T12:24:29,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/682178f8ee0c420386afd64a69add125 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125 2024-11-07T12:24:29,138 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125, entries=150, sequenceid=390, filesize=12.0 K 2024-11-07T12:24:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/265b2e263eee45d98c97321bdf6b16bd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd 2024-11-07T12:24:29,142 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd, entries=150, sequenceid=390, filesize=12.0 K 2024-11-07T12:24:29,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 847ms, sequenceid=390, compaction requested=true 2024-11-07T12:24:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:29,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T12:24:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T12:24:29,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T12:24:29,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-11-07T12:24:29,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0050 sec 2024-11-07T12:24:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T12:24:29,281 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:29,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T12:24:29,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:29,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:29,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:29,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T12:24:29,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:29,435 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:24:29,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62485aeb21a649538461f78262a34bd8 is 50, key is test_row_0/A:col10/1730982268336/Put/seqid=0 2024-11-07T12:24:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742151_1327 (size=12301) 2024-11-07T12:24:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. as already flushing 2024-11-07T12:24:29,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982329515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982329516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,594 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-07T12:24:29,594 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:29,594 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,594 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-07T12:24:29,594 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-07T12:24:29,594 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,594 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:29,594 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:29,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:29,594 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,594 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,595 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:29,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:29,595 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,598 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-07T12:24:29,598 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:29,598 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,598 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:29,599 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-07T12:24:29,599 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:29,599 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,599 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:29,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982329620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982329631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982329831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982329841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:29,842 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62485aeb21a649538461f78262a34bd8 2024-11-07T12:24:29,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/7b79482c7ea04b6cae09a20dd9b39d11 is 50, key is test_row_0/B:col10/1730982268336/Put/seqid=0 2024-11-07T12:24:29,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742152_1328 (size=12301) 2024-11-07T12:24:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:30,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982330141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982330152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:30,258 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/7b79482c7ea04b6cae09a20dd9b39d11 2024-11-07T12:24:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/7a2eda0413064cb299056750d0f82b0c is 50, key is test_row_0/C:col10/1730982268336/Put/seqid=0 2024-11-07T12:24:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742153_1329 (size=12301) 2024-11-07T12:24:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:30,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:24:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40464 deadline: 1730982330651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:30,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40410 deadline: 1730982330661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:24:30,675 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/7a2eda0413064cb299056750d0f82b0c 2024-11-07T12:24:30,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/62485aeb21a649538461f78262a34bd8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8 2024-11-07T12:24:30,720 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:24:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/7b79482c7ea04b6cae09a20dd9b39d11 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11 2024-11-07T12:24:30,724 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:24:30,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/7a2eda0413064cb299056750d0f82b0c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c 2024-11-07T12:24:30,728 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T12:24:30,728 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1293ms, sequenceid=406, compaction requested=true 2024-11-07T12:24:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T12:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T12:24:30,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T12:24:30,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-07T12:24:30,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4500 sec 2024-11-07T12:24:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T12:24:31,433 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:31,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:24:31,674 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-07T12:24:31,674 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-07T12:24:31,674 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:31,674 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:31,674 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:31,674 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:31,674 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:31,675 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:31,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:31,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:31,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/fd9e7d4699f24ab7b812af47935e58a0 is 50, key is test_row_0/A:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:31,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742154_1330 (size=12301) 2024-11-07T12:24:32,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/fd9e7d4699f24ab7b812af47935e58a0 2024-11-07T12:24:32,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/d15778e1a5384f1b8eb570029ccabbdc is 50, key is test_row_0/B:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:32,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742155_1331 (size=12301) 2024-11-07T12:24:32,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/d15778e1a5384f1b8eb570029ccabbdc 2024-11-07T12:24:32,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/9e3eb46f6d7b4e73a6aaac553200e7cc is 50, key is test_row_0/C:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742156_1332 (size=12301) 2024-11-07T12:24:32,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/9e3eb46f6d7b4e73a6aaac553200e7cc 2024-11-07T12:24:32,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/fd9e7d4699f24ab7b812af47935e58a0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0 2024-11-07T12:24:32,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:24:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/d15778e1a5384f1b8eb570029ccabbdc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc 2024-11-07T12:24:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:24:32,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/9e3eb46f6d7b4e73a6aaac553200e7cc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc 2024-11-07T12:24:32,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T12:24:32,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1277ms, sequenceid=428, compaction requested=true 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:32,951 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5c03ab2185aa77a0f7f7a6bc80ad010c:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:32,951 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64763 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/B is initiating minor compaction (all files) 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/A is initiating minor compaction (all files) 2024-11-07T12:24:32,953 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/B in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:32,953 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/A in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:32,953 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3b52ba1c836b4a27ab098fcfe049cc96, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=63.2 K 2024-11-07T12:24:32,953 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=60.9 K 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b52ba1c836b4a27ab098fcfe049cc96, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f4ddcdd1e9bf4b27b3f2c3be49d27a83, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c11a74df6cb54d0a818efaff07c38fe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730982266512 2024-11-07T12:24:32,953 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54ea20aafba439bbd4a854354a5b9b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730982266498 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef7b2fd3e304ef4a0256f3cc7622f76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1730982267176 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 682178f8ee0c420386afd64a69add125, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1730982267176 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62485aeb21a649538461f78262a34bd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982268331 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b79482c7ea04b6cae09a20dd9b39d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982268331 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d15778e1a5384f1b8eb570029ccabbdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730982269506 2024-11-07T12:24:32,954 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9e7d4699f24ab7b812af47935e58a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730982269506 2024-11-07T12:24:32,963 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:32,963 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:32,963 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/56af1532d4184eb99e5df3f7fa32b2ea is 50, key is test_row_0/A:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:32,963 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/fae0c7aed1354c0b9221aeb37ed2ae95 is 50, key is test_row_0/B:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742158_1334 (size=13289) 2024-11-07T12:24:32,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742157_1333 (size=13289) 2024-11-07T12:24:32,971 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/fae0c7aed1354c0b9221aeb37ed2ae95 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/fae0c7aed1354c0b9221aeb37ed2ae95 2024-11-07T12:24:32,975 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/B of 5c03ab2185aa77a0f7f7a6bc80ad010c into fae0c7aed1354c0b9221aeb37ed2ae95(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:32,975 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:32,975 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/B, priority=11, startTime=1730982272951; duration=0sec 2024-11-07T12:24:32,975 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:32,975 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:B 2024-11-07T12:24:32,975 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T12:24:32,976 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T12:24:32,976 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): 5c03ab2185aa77a0f7f7a6bc80ad010c/C is initiating minor compaction (all files) 2024-11-07T12:24:32,977 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5c03ab2185aa77a0f7f7a6bc80ad010c/C in TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:32,977 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/47f1e5afdbc945569d386daa74f68f1a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp, totalSize=60.9 K 2024-11-07T12:24:32,977 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f1e5afdbc945569d386daa74f68f1a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1730982266153 2024-11-07T12:24:32,977 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ad4c88a08d2b401f9362840ca9bb7537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730982266512 2024-11-07T12:24:32,977 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 265b2e263eee45d98c97321bdf6b16bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1730982267176 2024-11-07T12:24:32,978 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2eda0413064cb299056750d0f82b0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730982268331 2024-11-07T12:24:32,978 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e3eb46f6d7b4e73a6aaac553200e7cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1730982269506 2024-11-07T12:24:32,986 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5c03ab2185aa77a0f7f7a6bc80ad010c#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:32,986 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/4cb1df66344e4d12ade3180aedf0e91d is 50, key is test_row_0/C:col10/1730982269506/Put/seqid=0 2024-11-07T12:24:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742159_1335 (size=13289) 2024-11-07T12:24:33,374 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/56af1532d4184eb99e5df3f7fa32b2ea as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/56af1532d4184eb99e5df3f7fa32b2ea 2024-11-07T12:24:33,381 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/A of 5c03ab2185aa77a0f7f7a6bc80ad010c into 56af1532d4184eb99e5df3f7fa32b2ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:33,381 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:33,381 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/A, priority=11, startTime=1730982272951; duration=0sec 2024-11-07T12:24:33,381 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:33,381 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:A 2024-11-07T12:24:33,397 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/4cb1df66344e4d12ade3180aedf0e91d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4cb1df66344e4d12ade3180aedf0e91d 2024-11-07T12:24:33,403 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5c03ab2185aa77a0f7f7a6bc80ad010c/C of 5c03ab2185aa77a0f7f7a6bc80ad010c into 4cb1df66344e4d12ade3180aedf0e91d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:33,403 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: 2024-11-07T12:24:33,403 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., storeName=5c03ab2185aa77a0f7f7a6bc80ad010c/C, priority=11, startTime=1730982272951; duration=0sec 2024-11-07T12:24:33,403 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:33,404 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5c03ab2185aa77a0f7f7a6bc80ad010c:C 2024-11-07T12:24:34,267 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-07T12:24:34,267 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:34,267 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,267 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-07T12:24:34,267 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,267 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:34,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:34,268 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,268 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:34,302 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-07T12:24:34,302 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:34,302 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,302 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3641 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3607 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3606 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3637 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3608 2024-11-07T12:24:34,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:24:34,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:24:34,303 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:24:34,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:34,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:34,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:24:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:34,308 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982274308"}]},"ts":"1730982274308"} 2024-11-07T12:24:34,310 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:24:34,310 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:24:34,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:24:34,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, UNASSIGN}] 2024-11-07T12:24:34,314 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, UNASSIGN 2024-11-07T12:24:34,315 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5c03ab2185aa77a0f7f7a6bc80ad010c, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:34,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, UNASSIGN because future has completed 2024-11-07T12:24:34,317 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:24:34,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:34,471 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:34,472 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:24:34,472 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 5c03ab2185aa77a0f7f7a6bc80ad010c, disabling compactions & flushes 2024-11-07T12:24:34,472 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:34,472 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:34,472 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. after waiting 0 ms 2024-11-07T12:24:34,472 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:34,473 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T12:24:34,480 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/836af2f98c064215b86d5916442849ef is 50, key is test_row_0/A:col10/1730982274301/Put/seqid=0 2024-11-07T12:24:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742160_1336 (size=9857) 2024-11-07T12:24:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:34,887 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/836af2f98c064215b86d5916442849ef 2024-11-07T12:24:34,896 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/f9d62b5a95fc470b9c4738daf56ee41f is 50, key is test_row_0/B:col10/1730982274301/Put/seqid=0 2024-11-07T12:24:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742161_1337 (size=9857) 2024-11-07T12:24:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:35,301 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/f9d62b5a95fc470b9c4738daf56ee41f 2024-11-07T12:24:35,312 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/630a2bf4a8124318a585f0c6cd0b13fb is 50, key is test_row_0/C:col10/1730982274301/Put/seqid=0 2024-11-07T12:24:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742162_1338 (size=9857) 2024-11-07T12:24:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:35,718 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/630a2bf4a8124318a585f0c6cd0b13fb 2024-11-07T12:24:35,728 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/A/836af2f98c064215b86d5916442849ef as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/836af2f98c064215b86d5916442849ef 2024-11-07T12:24:35,732 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/836af2f98c064215b86d5916442849ef, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:24:35,733 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/B/f9d62b5a95fc470b9c4738daf56ee41f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f9d62b5a95fc470b9c4738daf56ee41f 2024-11-07T12:24:35,738 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f9d62b5a95fc470b9c4738daf56ee41f, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:24:35,739 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/.tmp/C/630a2bf4a8124318a585f0c6cd0b13fb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/630a2bf4a8124318a585f0c6cd0b13fb 2024-11-07T12:24:35,743 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/630a2bf4a8124318a585f0c6cd0b13fb, entries=100, sequenceid=437, filesize=9.6 K 2024-11-07T12:24:35,744 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1272ms, sequenceid=437, compaction requested=false 2024-11-07T12:24:35,745 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/93f0740bcd9347afa0dbfc03ecf9527c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/bb2c69b0a81a42d5a4aead52088c5a42, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/99d0caeec36e4272aafa2df46eb19493, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/554bd880bf4a44c786888f1a3d5f8f85, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5a32851448784636abc1d8d0c9ccaee6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/36830c0f74c14c2f945fc1bfb0efa803, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3b52ba1c836b4a27ab098fcfe049cc96, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0] to archive 2024-11-07T12:24:35,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:35,748 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/108cbc1c198f4208bbd7c8e604ef1ee8 2024-11-07T12:24:35,749 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/c92e6d77af414bef877382ab69d9997e 2024-11-07T12:24:35,750 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/e028815ca14042bdaf206e842fa4f890 2024-11-07T12:24:35,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/93f0740bcd9347afa0dbfc03ecf9527c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/93f0740bcd9347afa0dbfc03ecf9527c 2024-11-07T12:24:35,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62a46a3c9ab0439586537eb91616b6a0 2024-11-07T12:24:35,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/b6c8bc777ba94364ac4f972e4cc6d174 2024-11-07T12:24:35,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/bb2c69b0a81a42d5a4aead52088c5a42 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/bb2c69b0a81a42d5a4aead52088c5a42 2024-11-07T12:24:35,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/76c233565d0c40ceb7583fc15b7b6b9f 2024-11-07T12:24:35,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/056f69d739d547b8bbbb14c46b464cd4 2024-11-07T12:24:35,755 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/99d0caeec36e4272aafa2df46eb19493 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/99d0caeec36e4272aafa2df46eb19493 2024-11-07T12:24:35,756 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3151d97d144242179b795c192279ae22 2024-11-07T12:24:35,757 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/30f08371d9d54639ac71a071700b69e9 2024-11-07T12:24:35,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/232e663e412a49d6970271dc25d67912 2024-11-07T12:24:35,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/11ebbb7834854ea39cf14c63cedbf5a2 2024-11-07T12:24:35,759 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/554bd880bf4a44c786888f1a3d5f8f85 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/554bd880bf4a44c786888f1a3d5f8f85 2024-11-07T12:24:35,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/00db73e8d8114ccebb34c7280bd2094a 2024-11-07T12:24:35,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/015683a06f564ffeac14ef95557dd973 2024-11-07T12:24:35,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5a32851448784636abc1d8d0c9ccaee6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5a32851448784636abc1d8d0c9ccaee6 2024-11-07T12:24:35,762 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/6d5a4f5fdcc542f4b3aa0339cd729400 2024-11-07T12:24:35,763 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/82e2bb2529124132886184187f0c77e1 2024-11-07T12:24:35,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/36830c0f74c14c2f945fc1bfb0efa803 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/36830c0f74c14c2f945fc1bfb0efa803 2024-11-07T12:24:35,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/749c8862386748f1950129e82fadf47c 2024-11-07T12:24:35,765 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/2bc45b7a596d4290a666c92ae7503107 2024-11-07T12:24:35,766 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/1faf55b29e794704a7305f1f7253c775 2024-11-07T12:24:35,767 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3b52ba1c836b4a27ab098fcfe049cc96 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/3b52ba1c836b4a27ab098fcfe049cc96 2024-11-07T12:24:35,767 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/a54ea20aafba439bbd4a854354a5b9b3 2024-11-07T12:24:35,768 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/5ef7b2fd3e304ef4a0256f3cc7622f76 2024-11-07T12:24:35,769 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/62485aeb21a649538461f78262a34bd8 2024-11-07T12:24:35,770 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/fd9e7d4699f24ab7b812af47935e58a0 2024-11-07T12:24:35,771 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/77ee375092984475a93c83edb399ca63, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/252221afcbb64b6ca1ba2a7cdfc934f7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8024f3ee85614cf6a6c029ff449ac975, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/da3ffde3c15243b09eef6ab2b162ee18, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/3ce0f463c3444726b26b3214144c159d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/ecb2405fd8374d72bdb1e8b9b49ddd57, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc] to archive 2024-11-07T12:24:35,772 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:35,773 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/21bbd928637545d4acffc8b422f97718 2024-11-07T12:24:35,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c1d770461a8242999d76621df1abe1a6 2024-11-07T12:24:35,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/77ee375092984475a93c83edb399ca63 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/77ee375092984475a93c83edb399ca63 2024-11-07T12:24:35,775 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/04714875b0a44f6d98d7e5631abd7915 2024-11-07T12:24:35,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/2eeefe8383674ebc8f8d8509bc291e41 2024-11-07T12:24:35,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/252221afcbb64b6ca1ba2a7cdfc934f7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/252221afcbb64b6ca1ba2a7cdfc934f7 2024-11-07T12:24:35,777 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/4ac96dadcaf243b6bd30b78e24dafbf9 2024-11-07T12:24:35,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/409978e8104941cea73118fbd5e01943 2024-11-07T12:24:35,779 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/1fbab9ebbe8a4143ae55fc6aa12854cc 2024-11-07T12:24:35,780 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8024f3ee85614cf6a6c029ff449ac975 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8024f3ee85614cf6a6c029ff449ac975 2024-11-07T12:24:35,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/61cac69e453d4c0b966cb5ee0ba7778f 2024-11-07T12:24:35,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/8a2941f1b01349f5b5fdf855b2e05804 2024-11-07T12:24:35,782 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e58fe55590e14041a028b92e19af6a60 2024-11-07T12:24:35,783 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/da3ffde3c15243b09eef6ab2b162ee18 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/da3ffde3c15243b09eef6ab2b162ee18 2024-11-07T12:24:35,784 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e86ef91851194c1d803c3da4eb09a197 2024-11-07T12:24:35,784 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/43477082b9b64e429d0620ea46d05890 2024-11-07T12:24:35,785 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/3ce0f463c3444726b26b3214144c159d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/3ce0f463c3444726b26b3214144c159d 2024-11-07T12:24:35,786 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e3098bc8c187423a91076e669d09ad0a 2024-11-07T12:24:35,786 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/caed363642ff420daacce1c4c450d4e9 2024-11-07T12:24:35,787 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/ecb2405fd8374d72bdb1e8b9b49ddd57 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/ecb2405fd8374d72bdb1e8b9b49ddd57 2024-11-07T12:24:35,788 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/003453078c5f4ae6ac0bb7f62830d17b 2024-11-07T12:24:35,789 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e013e62a013a409996d75e979a87a470 2024-11-07T12:24:35,790 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/aad6b41ee3224d688d0298f4bb7c05ff 2024-11-07T12:24:35,790 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f4ddcdd1e9bf4b27b3f2c3be49d27a83 2024-11-07T12:24:35,791 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/e46f4fe11bad412599361a77a1407c9f 2024-11-07T12:24:35,792 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/c11a74df6cb54d0a818efaff07c38fe3 2024-11-07T12:24:35,793 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/682178f8ee0c420386afd64a69add125 2024-11-07T12:24:35,794 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/7b79482c7ea04b6cae09a20dd9b39d11 2024-11-07T12:24:35,795 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/d15778e1a5384f1b8eb570029ccabbdc 2024-11-07T12:24:35,796 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/a8fa570f95ad48dfa54d8efb8bc545fc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11db0afa0eb94f3f8f98978d6fc01bfa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/06776dc3bb814359b673d12db49f568f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11561f27a9c140c79c83194e45fc29df, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/58a088004833462b9132857f1b37e08b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/37791aed1fbd49238b91d19a0902fe0d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/47f1e5afdbc945569d386daa74f68f1a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc] to archive 2024-11-07T12:24:35,797 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:24:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f9e4ff74e97c4c08aa06cdada1932337 2024-11-07T12:24:35,799 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/98c60ef7226f46d3bf183d332183d135 2024-11-07T12:24:35,800 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/a8fa570f95ad48dfa54d8efb8bc545fc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/a8fa570f95ad48dfa54d8efb8bc545fc 2024-11-07T12:24:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2be015b9f4554f678715de8e5f0727d0 2024-11-07T12:24:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/5017091b7e444691818dc5e253e5f0e3 2024-11-07T12:24:35,802 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11db0afa0eb94f3f8f98978d6fc01bfa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11db0afa0eb94f3f8f98978d6fc01bfa 2024-11-07T12:24:35,803 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/394fc0df07834f3d968be6fa30a840aa 2024-11-07T12:24:35,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/22d61b5eec9448cf9903a2902c39d1b3 2024-11-07T12:24:35,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f2670661386a4e23a44a05309f9fe0c3 2024-11-07T12:24:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/06776dc3bb814359b673d12db49f568f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/06776dc3bb814359b673d12db49f568f 2024-11-07T12:24:35,806 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4c2068f7fe444f34b0e9609867b18dd3 2024-11-07T12:24:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f81103be32254c8ca123e641354b359b 2024-11-07T12:24:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/c558127d70934c4fad24b9f034d41240 2024-11-07T12:24:35,808 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11561f27a9c140c79c83194e45fc29df to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/11561f27a9c140c79c83194e45fc29df 2024-11-07T12:24:35,809 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/6815c6ed8de045c0a01d2f232b9e50e7 2024-11-07T12:24:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/f350c78cd9034b99835a09b28a22d0da 2024-11-07T12:24:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/58a088004833462b9132857f1b37e08b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/58a088004833462b9132857f1b37e08b 2024-11-07T12:24:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ecb5db1ecc8c41969379df5c07c93b40 2024-11-07T12:24:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/b96b3337c1f34499b748b72efad61df9 2024-11-07T12:24:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/37791aed1fbd49238b91d19a0902fe0d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/37791aed1fbd49238b91d19a0902fe0d 2024-11-07T12:24:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/df31869c79e94d779c9856f5ac307957 2024-11-07T12:24:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/2bf471f8c3c1414da641dd8786a8157f 2024-11-07T12:24:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/bf16e82fb54e4325811d7a4c08ee1df0 2024-11-07T12:24:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/47f1e5afdbc945569d386daa74f68f1a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/47f1e5afdbc945569d386daa74f68f1a 2024-11-07T12:24:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/e4f7ace8870e4d3989ff333ea1ff1aec 2024-11-07T12:24:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/ad4c88a08d2b401f9362840ca9bb7537 2024-11-07T12:24:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/265b2e263eee45d98c97321bdf6b16bd 2024-11-07T12:24:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/7a2eda0413064cb299056750d0f82b0c 2024-11-07T12:24:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/9e3eb46f6d7b4e73a6aaac553200e7cc 2024-11-07T12:24:35,823 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/recovered.edits/440.seqid, newMaxSeqId=440, maxSeqId=1 2024-11-07T12:24:35,824 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. 2024-11-07T12:24:35,824 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 5c03ab2185aa77a0f7f7a6bc80ad010c: Waiting for close lock at 1730982274472Running coprocessor pre-close hooks at 1730982274472Disabling compacts and flushes for region at 1730982274472Disabling writes for close at 1730982274472Obtaining lock to block concurrent updates at 1730982274473 (+1 ms)Preparing flush snapshotting stores in 5c03ab2185aa77a0f7f7a6bc80ad010c at 1730982274473Finished memstore snapshotting TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730982274473Flushing stores of TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c. at 1730982274474 (+1 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/A: creating writer at 1730982274475 (+1 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/A: appending metadata at 1730982274479 (+4 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/A: closing flushed file at 1730982274479Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/B: creating writer at 1730982274893 (+414 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/B: appending metadata at 1730982274896 (+3 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/B: closing flushed file at 1730982274896Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/C: creating writer at 1730982275307 (+411 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/C: appending metadata at 1730982275311 (+4 ms)Flushing 5c03ab2185aa77a0f7f7a6bc80ad010c/C: closing flushed file at 1730982275311Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53953696: reopening flushed file at 1730982275727 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d1777f5: reopening flushed file at 1730982275733 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58e24676: reopening flushed file at 1730982275738 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 5c03ab2185aa77a0f7f7a6bc80ad010c in 1272ms, sequenceid=437, compaction requested=false at 1730982275744 (+6 ms)Writing region close event to WAL at 1730982275820 (+76 ms)Running coprocessor post-close hooks at 1730982275824 (+4 ms)Closed at 1730982275824 2024-11-07T12:24:35,826 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:35,826 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5c03ab2185aa77a0f7f7a6bc80ad010c, regionState=CLOSED 2024-11-07T12:24:35,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:35,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T12:24:35,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 5c03ab2185aa77a0f7f7a6bc80ad010c, server=8944cbb06cd2,46221,1730982182218 in 1.5110 sec 2024-11-07T12:24:35,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-07T12:24:35,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5c03ab2185aa77a0f7f7a6bc80ad010c, UNASSIGN in 1.5180 sec 2024-11-07T12:24:35,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T12:24:35,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-07T12:24:35,835 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982275834"}]},"ts":"1730982275834"} 2024-11-07T12:24:35,836 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:24:35,836 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:24:35,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-07T12:24:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T12:24:36,451 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:36,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:24:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,453 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T12:24:36,454 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:36,458 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/recovered.edits] 2024-11-07T12:24:36,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/56af1532d4184eb99e5df3f7fa32b2ea to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/56af1532d4184eb99e5df3f7fa32b2ea 2024-11-07T12:24:36,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/836af2f98c064215b86d5916442849ef to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/A/836af2f98c064215b86d5916442849ef 2024-11-07T12:24:36,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f9d62b5a95fc470b9c4738daf56ee41f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/f9d62b5a95fc470b9c4738daf56ee41f 2024-11-07T12:24:36,464 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/fae0c7aed1354c0b9221aeb37ed2ae95 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/B/fae0c7aed1354c0b9221aeb37ed2ae95 2024-11-07T12:24:36,465 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4cb1df66344e4d12ade3180aedf0e91d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/4cb1df66344e4d12ade3180aedf0e91d 2024-11-07T12:24:36,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/630a2bf4a8124318a585f0c6cd0b13fb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/C/630a2bf4a8124318a585f0c6cd0b13fb 2024-11-07T12:24:36,469 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/recovered.edits/440.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c/recovered.edits/440.seqid 2024-11-07T12:24:36,469 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/5c03ab2185aa77a0f7f7a6bc80ad010c 2024-11-07T12:24:36,469 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:24:36,471 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,474 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:24:36,476 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:24:36,478 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,478 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:24:36,478 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982276478"}]},"ts":"9223372036854775807"} 2024-11-07T12:24:36,480 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:24:36,480 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5c03ab2185aa77a0f7f7a6bc80ad010c, NAME => 'TestAcidGuarantees,,1730982247384.5c03ab2185aa77a0f7f7a6bc80ad010c.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:24:36,480 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:24:36,480 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982276480"}]},"ts":"9223372036854775807"} 2024-11-07T12:24:36,482 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:24:36,482 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-07T12:24:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T12:24:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:24:36,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:36,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 242), OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=600 (was 683), ProcessCount=11 (was 11), AvailableMemoryMB=1492 (was 1987) 2024-11-07T12:24:36,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=600, ProcessCount=11, AvailableMemoryMB=1491 2024-11-07T12:24:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:24:36,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:24:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:36,587 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:24:36,587 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:36,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-07T12:24:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:36,588 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:24:36,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742163_1339 (size=959) 2024-11-07T12:24:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:36,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:24:37,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742164_1340 (size=53) 2024-11-07T12:24:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e34bd2129ac88e7686bed4817db84e03, disabling compactions & flushes 2024-11-07T12:24:37,409 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. after waiting 0 ms 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,409 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e34bd2129ac88e7686bed4817db84e03: Waiting for close lock at 1730982277409Disabling compacts and flushes for region at 1730982277409Disabling writes for close at 1730982277409Writing region close event to WAL at 1730982277409Closed at 1730982277409 2024-11-07T12:24:37,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:24:37,412 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982277411"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982277411"}]},"ts":"1730982277411"} 2024-11-07T12:24:37,415 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:24:37,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:24:37,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982277417"}]},"ts":"1730982277417"} 2024-11-07T12:24:37,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:24:37,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, ASSIGN}] 2024-11-07T12:24:37,423 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, ASSIGN 2024-11-07T12:24:37,424 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:24:37,575 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:37,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, ASSIGN because future has completed 2024-11-07T12:24:37,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:37,734 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,734 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:24:37,735 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,735 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:24:37,735 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,735 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,736 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,738 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName A 2024-11-07T12:24:37,738 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:37,738 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:37,738 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,739 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName B 2024-11-07T12:24:37,739 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:37,740 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:37,740 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,741 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName C 2024-11-07T12:24:37,741 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:37,742 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:37,742 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,742 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,743 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,743 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,743 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,744 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:24:37,745 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,746 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:24:37,746 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened e34bd2129ac88e7686bed4817db84e03; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61222776, jitterRate=-0.08770954608917236}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:24:37,746 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:37,747 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for e34bd2129ac88e7686bed4817db84e03: Running coprocessor pre-open hook at 1730982277735Writing region info on filesystem at 1730982277735Initializing all the Stores at 1730982277736 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982277736Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982277736Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982277736Cleaning up temporary data from old regions at 1730982277743 (+7 ms)Running coprocessor post-open hooks at 1730982277746 (+3 ms)Region opened successfully at 1730982277747 (+1 ms) 2024-11-07T12:24:37,748 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., pid=89, masterSystemTime=1730982277731 2024-11-07T12:24:37,749 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,749 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:37,750 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:37,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:37,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T12:24:37,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 in 174 msec 2024-11-07T12:24:37,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T12:24:37,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, ASSIGN in 334 msec 2024-11-07T12:24:37,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:24:37,758 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982277757"}]},"ts":"1730982277757"} 2024-11-07T12:24:37,759 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:24:37,761 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:24:37,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-07T12:24:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-07T12:24:38,731 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:38,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59596ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:38,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:38,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:38,734 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a33704d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:38,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:38,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af47d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:38,738 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:38,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:38,743 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:24:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:24:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6913258 2024-11-07T12:24:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:24:38,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:24:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:24:38,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:24:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:24:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742165_1341 (size=995) 2024-11-07T12:24:39,162 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:24:39,163 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:24:39,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:24:39,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE}] 2024-11-07T12:24:39,171 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE 2024-11-07T12:24:39,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE because future has completed 2024-11-07T12:24:39,173 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:24:39,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:39,325 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,326 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:24:39,326 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing e34bd2129ac88e7686bed4817db84e03, disabling compactions & flushes 2024-11-07T12:24:39,326 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,326 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,326 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. after waiting 0 ms 2024-11-07T12:24:39,326 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,329 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:24:39,330 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,330 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for e34bd2129ac88e7686bed4817db84e03: Waiting for close lock at 1730982279326Running coprocessor pre-close hooks at 1730982279326Disabling compacts and flushes for region at 1730982279326Disabling writes for close at 1730982279326Writing region close event to WAL at 1730982279326Running coprocessor post-close hooks at 1730982279330 (+4 ms)Closed at 1730982279330 2024-11-07T12:24:39,330 WARN [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: e34bd2129ac88e7686bed4817db84e03 to self. 2024-11-07T12:24:39,331 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,332 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=CLOSED 2024-11-07T12:24:39,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:39,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T12:24:39,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 in 161 msec 2024-11-07T12:24:39,336 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE; state=CLOSED, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=true 2024-11-07T12:24:39,487 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE because future has completed 2024-11-07T12:24:39,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:24:39,642 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,642 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:24:39,643 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,643 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:24:39,643 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,643 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,644 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,644 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName A 2024-11-07T12:24:39,645 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:39,646 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:39,646 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,647 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName B 2024-11-07T12:24:39,647 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:39,647 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:39,647 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,648 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e34bd2129ac88e7686bed4817db84e03 columnFamilyName C 2024-11-07T12:24:39,648 DEBUG [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:39,648 INFO [StoreOpener-e34bd2129ac88e7686bed4817db84e03-1 {}] regionserver.HStore(327): Store=e34bd2129ac88e7686bed4817db84e03/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:24:39,648 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,649 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,649 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,650 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,650 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,650 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:24:39,651 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,652 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened e34bd2129ac88e7686bed4817db84e03; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62883923, jitterRate=-0.06295652687549591}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:24:39,652 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,652 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for e34bd2129ac88e7686bed4817db84e03: Running coprocessor pre-open hook at 1730982279643Writing region info on filesystem at 1730982279643Initializing all the Stores at 1730982279644 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982279644Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982279644Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982279644Cleaning up temporary data from old regions at 1730982279650 (+6 ms)Running coprocessor post-open hooks at 1730982279652 (+2 ms)Region opened successfully at 1730982279652 2024-11-07T12:24:39,653 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., pid=94, masterSystemTime=1730982279640 2024-11-07T12:24:39,655 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,655 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=OPEN, openSeqNum=5, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:24:39,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-07T12:24:39,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 in 169 msec 2024-11-07T12:24:39,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T12:24:39,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, REOPEN/MOVE in 489 msec 2024-11-07T12:24:39,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T12:24:39,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-07T12:24:39,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-07T12:24:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T12:24:39,665 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:39,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4853c07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4822a2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7d3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aea0fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d96e357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b614aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216ebabc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,675 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f0952d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c87087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,677 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730694e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,679 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eca3b10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7679e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,680 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432b4b91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acff738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f259004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcaa59c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@768973e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526ed020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c26906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a1cdc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2ec5f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45141c4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3683273f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ee78e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,694 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1c1f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158dd69e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ee060c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:24:39,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:24:39,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:24:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:24:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:24:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e816b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:24:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:24:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:39,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:24:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc0d42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:24:39,700 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,700 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,700 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,701 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,701 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,701 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:39,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,703 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-07T12:24:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:39,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,707 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:39,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,708 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,710 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:39,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:39,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,712 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,716 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,716 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:24:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:24:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:24:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:39,719 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:39,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:24:39,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:24:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:24:39,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110706f87fef55214c2f988783a7d9c87006_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982279715/Put/seqid=0 2024-11-07T12:24:39,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742166_1342 (size=12154) 2024-11-07T12:24:39,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982339767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982339768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982339781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982339781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982339782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:39,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:39,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:39,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982339891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982339891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982339891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982339891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982339900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982340101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982340101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982340101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982340101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982340110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:40,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110706f87fef55214c2f988783a7d9c87006_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110706f87fef55214c2f988783a7d9c87006_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:40,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/103162e6e1c4449699c7ad51d3536f9e, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:40,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/103162e6e1c4449699c7ad51d3536f9e is 175, key is test_row_0/A:col10/1730982279715/Put/seqid=0 2024-11-07T12:24:40,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742167_1343 (size=30955) 2024-11-07T12:24:40,172 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/103162e6e1c4449699c7ad51d3536f9e 2024-11-07T12:24:40,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0e20f633763147dfbb13677eb9e329c4 is 50, key is test_row_0/B:col10/1730982279715/Put/seqid=0 2024-11-07T12:24:40,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742168_1344 (size=12001) 2024-11-07T12:24:40,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982340411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982340411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982340411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982340411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982340420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0e20f633763147dfbb13677eb9e329c4 2024-11-07T12:24:40,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/ebda12a300d9451bac2bb30fa9d34345 is 50, key is test_row_0/C:col10/1730982279715/Put/seqid=0 2024-11-07T12:24:40,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742169_1345 (size=12001) 2024-11-07T12:24:40,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982340921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982340921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982340921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:40,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982340930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982340930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/ebda12a300d9451bac2bb30fa9d34345 2024-11-07T12:24:41,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/103162e6e1c4449699c7ad51d3536f9e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e 2024-11-07T12:24:41,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e, entries=150, sequenceid=15, filesize=30.2 K 2024-11-07T12:24:41,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0e20f633763147dfbb13677eb9e329c4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4 2024-11-07T12:24:41,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:24:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/ebda12a300d9451bac2bb30fa9d34345 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345 2024-11-07T12:24:41,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T12:24:41,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for e34bd2129ac88e7686bed4817db84e03 in 1319ms, sequenceid=15, compaction requested=false 2024-11-07T12:24:41,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:41,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-07T12:24:41,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:41,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:24:41,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c321c056d4943b8ab98bfb1202c8af7_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982279763/Put/seqid=0 2024-11-07T12:24:41,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742170_1346 (size=12154) 2024-11-07T12:24:41,353 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:24:41,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:41,499 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c321c056d4943b8ab98bfb1202c8af7_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c321c056d4943b8ab98bfb1202c8af7_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:41,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/9e6cb0d906784bf68ebcbc243e52f4f3, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:41,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/9e6cb0d906784bf68ebcbc243e52f4f3 is 175, key is test_row_0/A:col10/1730982279763/Put/seqid=0 2024-11-07T12:24:41,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742171_1347 (size=30955) 2024-11-07T12:24:41,507 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/9e6cb0d906784bf68ebcbc243e52f4f3 2024-11-07T12:24:41,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/a90a330b694145e79b9661044d52675a is 50, key is test_row_0/B:col10/1730982279763/Put/seqid=0 2024-11-07T12:24:41,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742172_1348 (size=12001) 2024-11-07T12:24:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:41,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/a90a330b694145e79b9661044d52675a 2024-11-07T12:24:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/704aeeee955042d7b15a39d9a4ce39b3 is 50, key is test_row_0/C:col10/1730982279763/Put/seqid=0 2024-11-07T12:24:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:41,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742173_1349 (size=12001) 2024-11-07T12:24:41,944 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/704aeeee955042d7b15a39d9a4ce39b3 2024-11-07T12:24:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/9e6cb0d906784bf68ebcbc243e52f4f3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3 2024-11-07T12:24:41,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982341947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3, entries=150, sequenceid=42, filesize=30.2 K 2024-11-07T12:24:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/a90a330b694145e79b9661044d52675a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a 2024-11-07T12:24:41,957 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T12:24:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982341947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982341948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982341949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982341951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/704aeeee955042d7b15a39d9a4ce39b3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3 2024-11-07T12:24:41,958 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:41,962 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T12:24:41,963 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e34bd2129ac88e7686bed4817db84e03 in 878ms, sequenceid=42, compaction requested=false 2024-11-07T12:24:41,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:41,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:41,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-07T12:24:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-07T12:24:41,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-07T12:24:41,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2540 sec 2024-11-07T12:24:41,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2660 sec 2024-11-07T12:24:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:42,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:42,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:24:42,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110779bc874bb5f14e2394ce8431df314011_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742174_1350 (size=12154) 2024-11-07T12:24:42,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982342117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982342117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982342123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982342124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982342230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982342230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982342231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982342241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982342441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982342441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982342441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982342451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:42,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110779bc874bb5f14e2394ce8431df314011_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110779bc874bb5f14e2394ce8431df314011_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:42,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/831aad9e74104526acfe32fe648735d9, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:42,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/831aad9e74104526acfe32fe648735d9 is 175, key is test_row_0/A:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:42,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742175_1351 (size=30955) 2024-11-07T12:24:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982342760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982342760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982342761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982342771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:42,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/831aad9e74104526acfe32fe648735d9 2024-11-07T12:24:42,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/1eca09832adb47bd8510a093a3dbcf74 is 50, key is test_row_0/B:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742176_1352 (size=12001) 2024-11-07T12:24:43,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982343270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:43,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982343280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982343280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:43,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982343291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:43,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/1eca09832adb47bd8510a093a3dbcf74 2024-11-07T12:24:43,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/88e95fa5e48847018292e4c02c8da642 is 50, key is test_row_0/C:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742177_1353 (size=12001) 2024-11-07T12:24:43,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/88e95fa5e48847018292e4c02c8da642 2024-11-07T12:24:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/831aad9e74104526acfe32fe648735d9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9 2024-11-07T12:24:43,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9, entries=150, sequenceid=54, filesize=30.2 K 2024-11-07T12:24:43,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/1eca09832adb47bd8510a093a3dbcf74 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74 2024-11-07T12:24:43,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:24:43,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/88e95fa5e48847018292e4c02c8da642 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642 2024-11-07T12:24:43,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:24:43,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e34bd2129ac88e7686bed4817db84e03 in 1711ms, sequenceid=54, compaction requested=true 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:43,776 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:43,776 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:43,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:43,777 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:43,777 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:43,777 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=90.7 K 2024-11-07T12:24:43,777 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=35.2 K 2024-11-07T12:24:43,777 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9] 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e20f633763147dfbb13677eb9e329c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730982279715 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 103162e6e1c4449699c7ad51d3536f9e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730982279715 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a90a330b694145e79b9661044d52675a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982279763 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e6cb0d906784bf68ebcbc243e52f4f3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982279763 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831aad9e74104526acfe32fe648735d9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:43,777 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eca09832adb47bd8510a093a3dbcf74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:43,782 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:43,783 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:43,784 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2e72fddea58e4a16826cf2f0566a4c95 is 50, key is test_row_0/B:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:43,785 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107936f7c6557b7401b9c93b0ed50766967_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:43,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742178_1354 (size=12104) 2024-11-07T12:24:43,787 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107936f7c6557b7401b9c93b0ed50766967_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:43,787 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107936f7c6557b7401b9c93b0ed50766967_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742179_1355 (size=4469) 2024-11-07T12:24:43,792 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2e72fddea58e4a16826cf2f0566a4c95 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e72fddea58e4a16826cf2f0566a4c95 2024-11-07T12:24:43,792 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#298 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:43,793 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/f170c4f56c0a45db8cf93768384cd161 is 175, key is test_row_0/A:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:43,797 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into 2e72fddea58e4a16826cf2f0566a4c95(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:43,797 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:43,797 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982283776; duration=0sec 2024-11-07T12:24:43,797 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:43,797 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:43,797 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:43,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742180_1356 (size=31058) 2024-11-07T12:24:43,807 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:43,807 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:43,807 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:43,807 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=35.2 K 2024-11-07T12:24:43,807 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ebda12a300d9451bac2bb30fa9d34345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730982279715 2024-11-07T12:24:43,808 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 704aeeee955042d7b15a39d9a4ce39b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730982279763 2024-11-07T12:24:43,808 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 88e95fa5e48847018292e4c02c8da642, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:43,812 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/f170c4f56c0a45db8cf93768384cd161 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161 2024-11-07T12:24:43,817 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:43,817 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/7e4e05fe988441f4b86ebae8162a4fe4 is 50, key is test_row_0/C:col10/1730982282065/Put/seqid=0 2024-11-07T12:24:43,818 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into f170c4f56c0a45db8cf93768384cd161(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:43,818 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:43,818 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982283776; duration=0sec 2024-11-07T12:24:43,818 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:43,818 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:43,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742181_1357 (size=12104) 2024-11-07T12:24:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-07T12:24:43,862 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:43,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-07T12:24:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:43,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:43,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:43,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:43,965 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:43,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:24:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:43,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107753ca431e9d44b3ab3f8f2bb367a9f0d_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982282116/Put/seqid=0 2024-11-07T12:24:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742182_1358 (size=14594) 2024-11-07T12:24:43,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:43,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107753ca431e9d44b3ab3f8f2bb367a9f0d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107753ca431e9d44b3ab3f8f2bb367a9f0d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:43,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5579467f305248f68465b8ba1dcec59f, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:43,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5579467f305248f68465b8ba1dcec59f is 175, key is test_row_0/A:col10/1730982282116/Put/seqid=0 2024-11-07T12:24:43,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742183_1359 (size=39549) 2024-11-07T12:24:43,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5579467f305248f68465b8ba1dcec59f 2024-11-07T12:24:43,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6c09406974bb4306a5982f2e45173587 is 50, key is test_row_0/B:col10/1730982282116/Put/seqid=0 2024-11-07T12:24:43,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742184_1360 (size=12001) 2024-11-07T12:24:44,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:24:44,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:44,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982344022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982344131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:24:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:44,230 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/7e4e05fe988441f4b86ebae8162a4fe4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7e4e05fe988441f4b86ebae8162a4fe4 2024-11-07T12:24:44,234 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into 7e4e05fe988441f4b86ebae8162a4fe4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:44,234 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:44,234 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982283776; duration=0sec 2024-11-07T12:24:44,235 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:44,235 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982344291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,296 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982344301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,305 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982344301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,308 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982344310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,316 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:24:44,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982344341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6c09406974bb4306a5982f2e45173587 2024-11-07T12:24:44,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/7971d8f48c044752b1d7512631b7ddb3 is 50, key is test_row_0/C:col10/1730982282116/Put/seqid=0 2024-11-07T12:24:44,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742185_1361 (size=12001) 2024-11-07T12:24:44,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/7971d8f48c044752b1d7512631b7ddb3 2024-11-07T12:24:44,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5579467f305248f68465b8ba1dcec59f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f 2024-11-07T12:24:44,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f, entries=200, sequenceid=81, filesize=38.6 K 2024-11-07T12:24:44,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6c09406974bb4306a5982f2e45173587 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587 2024-11-07T12:24:44,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T12:24:44,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/7971d8f48c044752b1d7512631b7ddb3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3 2024-11-07T12:24:44,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T12:24:44,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e34bd2129ac88e7686bed4817db84e03 in 466ms, sequenceid=81, compaction requested=false 2024-11-07T12:24:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:44,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T12:24:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:44,477 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:24:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079042f988bbad40c697f0e4b53d2393eb_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982284020/Put/seqid=0 2024-11-07T12:24:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742186_1362 (size=12154) 2024-11-07T12:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:44,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982344804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:44,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079042f988bbad40c697f0e4b53d2393eb_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079042f988bbad40c697f0e4b53d2393eb_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/361abc196bbb48c294c1ec6f5402019c, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:44,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/361abc196bbb48c294c1ec6f5402019c is 175, key is test_row_0/A:col10/1730982284020/Put/seqid=0 2024-11-07T12:24:44,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742187_1363 (size=30955) 2024-11-07T12:24:44,898 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/361abc196bbb48c294c1ec6f5402019c 2024-11-07T12:24:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0f81d7b2c20548edbdcf8beb60d02583 is 50, key is test_row_0/B:col10/1730982284020/Put/seqid=0 2024-11-07T12:24:44,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742188_1364 (size=12001) 2024-11-07T12:24:44,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0f81d7b2c20548edbdcf8beb60d02583 2024-11-07T12:24:44,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/f0d0bef0ae9148d58a6ad43ce611c810 is 50, key is test_row_0/C:col10/1730982284020/Put/seqid=0 2024-11-07T12:24:44,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742189_1365 (size=12001) 2024-11-07T12:24:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982344920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982345130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:45,321 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/f0d0bef0ae9148d58a6ad43ce611c810 2024-11-07T12:24:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/361abc196bbb48c294c1ec6f5402019c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c 2024-11-07T12:24:45,338 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c, entries=150, sequenceid=93, filesize=30.2 K 2024-11-07T12:24:45,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/0f81d7b2c20548edbdcf8beb60d02583 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583 2024-11-07T12:24:45,344 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:24:45,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/f0d0bef0ae9148d58a6ad43ce611c810 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810 2024-11-07T12:24:45,348 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T12:24:45,349 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e34bd2129ac88e7686bed4817db84e03 in 872ms, sequenceid=93, compaction requested=true 2024-11-07T12:24:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:45,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-07T12:24:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-07T12:24:45,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-07T12:24:45,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-07T12:24:45,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4890 sec 2024-11-07T12:24:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:45,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:45,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:24:45,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078d14acca2d4846c59c34b05d721d366f_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742190_1366 (size=14594) 2024-11-07T12:24:45,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982345503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:45,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982345611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982345831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:45,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:45,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078d14acca2d4846c59c34b05d721d366f_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d14acca2d4846c59c34b05d721d366f_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:45,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/4409c5b6cdc041a7a904494ac979f36c, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:45,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/4409c5b6cdc041a7a904494ac979f36c is 175, key is test_row_0/A:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742191_1367 (size=39549) 2024-11-07T12:24:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T12:24:46,011 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:46,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-07T12:24:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:24:46,012 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:46,013 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:46,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:24:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982346152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:24:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/4409c5b6cdc041a7a904494ac979f36c 2024-11-07T12:24:46,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/85f29e7765044c3aa75628ae165c3031 is 50, key is test_row_0/B:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982346311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982346311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:24:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,318 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:46,319 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:24:46,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742192_1368 (size=12001) 2024-11-07T12:24:46,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/85f29e7765044c3aa75628ae165c3031 2024-11-07T12:24:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982346331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982346331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,345 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b23768a9246d49eaac7f0056b17911c2 is 50, key is test_row_0/C:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:46,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742193_1369 (size=12001) 2024-11-07T12:24:46,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b23768a9246d49eaac7f0056b17911c2 2024-11-07T12:24:46,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/4409c5b6cdc041a7a904494ac979f36c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c 2024-11-07T12:24:46,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c, entries=200, sequenceid=119, filesize=38.6 K 2024-11-07T12:24:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/85f29e7765044c3aa75628ae165c3031 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031 2024-11-07T12:24:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b23768a9246d49eaac7f0056b17911c2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2 2024-11-07T12:24:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e34bd2129ac88e7686bed4817db84e03 in 935ms, sequenceid=119, compaction requested=true 2024-11-07T12:24:46,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:46,392 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:46,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,393 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:46,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:46,394 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,394 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,394 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=137.8 K 2024-11-07T12:24:46,394 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e72fddea58e4a16826cf2f0566a4c95, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=47.0 K 2024-11-07T12:24:46,394 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c] 2024-11-07T12:24:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e72fddea58e4a16826cf2f0566a4c95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f170c4f56c0a45db8cf93768384cd161, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:46,394 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c09406974bb4306a5982f2e45173587, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730982282116 2024-11-07T12:24:46,395 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5579467f305248f68465b8ba1dcec59f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730982282116 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,396 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361abc196bbb48c294c1ec6f5402019c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982283985 2024-11-07T12:24:46,396 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f81d7b2c20548edbdcf8beb60d02583, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982283985 2024-11-07T12:24:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,396 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4409c5b6cdc041a7a904494ac979f36c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:46,396 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f29e7765044c3aa75628ae165c3031, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,404 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,406 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110799a626bb0e84430486d6322509bc6d17_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,408 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,408 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110799a626bb0e84430486d6322509bc6d17_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:46,409 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110799a626bb0e84430486d6322509bc6d17_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,409 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/c83b0f333205416ea76ccc42126fed3c is 50, key is test_row_0/B:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742195_1371 (size=12241) 2024-11-07T12:24:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742194_1370 (size=4469) 2024-11-07T12:24:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,432 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#309 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,432 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/0cefa69e34d24fe1a3213bd8e9393755 is 175, key is test_row_0/A:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742196_1372 (size=31195) 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,453 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/0cefa69e34d24fe1a3213bd8e9393755 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755 2024-11-07T12:24:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,457 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into 0cefa69e34d24fe1a3213bd8e9393755(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:46,457 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:46,457 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=12, startTime=1730982286392; duration=0sec 2024-11-07T12:24:46,458 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:46,458 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:46,458 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:24:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,461 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:24:46,461 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:46,461 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,461 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7e4e05fe988441f4b86ebae8162a4fe4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=47.0 K 2024-11-07T12:24:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,461 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e4e05fe988441f4b86ebae8162a4fe4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982282063 2024-11-07T12:24:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,462 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7971d8f48c044752b1d7512631b7ddb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730982282116 2024-11-07T12:24:46,462 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d0bef0ae9148d58a6ad43ce611c810, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730982283985 2024-11-07T12:24:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,462 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b23768a9246d49eaac7f0056b17911c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,470 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/c6a173fbfd714b9d9b21ce2dd3e7bcda is 50, key is test_row_0/C:col10/1730982285455/Put/seqid=0 2024-11-07T12:24:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T12:24:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a6d6a6b6c504ff5b0282e77e28cb6b2_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982285492/Put/seqid=0 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742198_1374 (size=9714) 2024-11-07T12:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742197_1373 (size=12241) 2024-11-07T12:24:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a6d6a6b6c504ff5b0282e77e28cb6b2_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a6d6a6b6c504ff5b0282e77e28cb6b2_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d48e77c2c78c4996986e0fb8214c3715, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d48e77c2c78c4996986e0fb8214c3715 is 175, key is test_row_0/A:col10/1730982285492/Put/seqid=0 2024-11-07T12:24:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742199_1375 (size=22361) 2024-11-07T12:24:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,524 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d48e77c2c78c4996986e0fb8214c3715 2024-11-07T12:24:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/647b4a486e99424eaf6e357b1f3f309c is 50, key is test_row_0/B:col10/1730982285492/Put/seqid=0 2024-11-07T12:24:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742200_1376 (size=9657) 2024-11-07T12:24:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,542 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/647b4a486e99424eaf6e357b1f3f309c 2024-11-07T12:24:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6f91da6f9b6444bcad17721139edefbc is 50, key is test_row_0/C:col10/1730982285492/Put/seqid=0 2024-11-07T12:24:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742201_1377 (size=9657) 2024-11-07T12:24:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:24:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:46,827 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/c83b0f333205416ea76ccc42126fed3c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/c83b0f333205416ea76ccc42126fed3c 2024-11-07T12:24:46,832 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into c83b0f333205416ea76ccc42126fed3c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:46,832 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:46,832 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=12, startTime=1730982286393; duration=0sec 2024-11-07T12:24:46,832 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:46,832 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:46,919 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/c6a173fbfd714b9d9b21ce2dd3e7bcda as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/c6a173fbfd714b9d9b21ce2dd3e7bcda 2024-11-07T12:24:46,924 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into c6a173fbfd714b9d9b21ce2dd3e7bcda(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:46,924 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:46,924 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=12, startTime=1730982286393; duration=0sec 2024-11-07T12:24:46,924 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:46,924 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:46,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982346962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:46,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6f91da6f9b6444bcad17721139edefbc 2024-11-07T12:24:46,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d48e77c2c78c4996986e0fb8214c3715 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715 2024-11-07T12:24:46,983 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715, entries=100, sequenceid=130, filesize=21.8 K 2024-11-07T12:24:46,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/647b4a486e99424eaf6e357b1f3f309c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c 2024-11-07T12:24:46,988 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c, entries=100, sequenceid=130, filesize=9.4 K 2024-11-07T12:24:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6f91da6f9b6444bcad17721139edefbc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc 2024-11-07T12:24:46,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc, entries=100, sequenceid=130, filesize=9.4 K 2024-11-07T12:24:46,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e34bd2129ac88e7686bed4817db84e03 in 521ms, sequenceid=130, compaction requested=false 2024-11-07T12:24:46,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:46,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:46,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-07T12:24:46,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-07T12:24:46,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T12:24:46,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 983 msec 2024-11-07T12:24:46,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 987 msec 2024-11-07T12:24:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:47,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:24:47,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c98cf25949a246d99eaf5f6f23d523a2_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742202_1378 (size=14794) 2024-11-07T12:24:47,122 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,126 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c98cf25949a246d99eaf5f6f23d523a2_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c98cf25949a246d99eaf5f6f23d523a2_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:47,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d0048aa44492412a8e6bc181e1c4a205, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d0048aa44492412a8e6bc181e1c4a205 is 175, key is test_row_0/A:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982347125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:47,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742203_1379 (size=39749) 2024-11-07T12:24:47,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d0048aa44492412a8e6bc181e1c4a205 2024-11-07T12:24:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T12:24:47,150 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:47,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-07T12:24:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:24:47,152 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:47,153 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:47,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:47,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2c95e08f67bc47868663ad1dbb2fa711 is 50, key is test_row_0/B:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742204_1380 (size=12151) 2024-11-07T12:24:47,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2c95e08f67bc47868663ad1dbb2fa711 2024-11-07T12:24:47,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/0e3e4d73c7cf48e882bdd5775e622f7e is 50, key is test_row_0/C:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742205_1381 (size=12151) 2024-11-07T12:24:47,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/0e3e4d73c7cf48e882bdd5775e622f7e 2024-11-07T12:24:47,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d0048aa44492412a8e6bc181e1c4a205 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205 2024-11-07T12:24:47,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205, entries=200, sequenceid=159, filesize=38.8 K 2024-11-07T12:24:47,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2c95e08f67bc47868663ad1dbb2fa711 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711 2024-11-07T12:24:47,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T12:24:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/0e3e4d73c7cf48e882bdd5775e622f7e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e 2024-11-07T12:24:47,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T12:24:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e34bd2129ac88e7686bed4817db84e03 in 134ms, sequenceid=159, compaction requested=true 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:47,220 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:24:47,220 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,221 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:47,221 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:47,221 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:47,221 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,221 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,221 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,221 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=91.1 K 2024-11-07T12:24:47,221 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/c83b0f333205416ea76ccc42126fed3c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=33.3 K 2024-11-07T12:24:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205] 2024-11-07T12:24:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83b0f333205416ea76ccc42126fed3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647b4a486e99424eaf6e357b1f3f309c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730982285492 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cefa69e34d24fe1a3213bd8e9393755, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d48e77c2c78c4996986e0fb8214c3715, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730982285492 2024-11-07T12:24:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,222 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c95e08f67bc47868663ad1dbb2fa711, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,223 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d0048aa44492412a8e6bc181e1c4a205, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,229 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,242 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:47,243 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/fa6d37a0349742ad9fbf0f601d7a4deb is 50, key is test_row_0/B:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,246 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c56cce19349a4061ba5d41c5f526b80a_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,248 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c56cce19349a4061ba5d41c5f526b80a_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,248 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c56cce19349a4061ba5d41c5f526b80a_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:24:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742206_1382 (size=12493) 2024-11-07T12:24:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,303 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/fa6d37a0349742ad9fbf0f601d7a4deb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fa6d37a0349742ad9fbf0f601d7a4deb 2024-11-07T12:24:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T12:24:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,307 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:24:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,312 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into fa6d37a0349742ad9fbf0f601d7a4deb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:47,312 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982287220; duration=0sec 2024-11-07T12:24:47,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:47,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:47,312 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,313 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:47,313 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:47,313 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,313 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/c6a173fbfd714b9d9b21ce2dd3e7bcda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=33.3 K 2024-11-07T12:24:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742207_1383 (size=4469) 2024-11-07T12:24:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,321 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a173fbfd714b9d9b21ce2dd3e7bcda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1730982284759 2024-11-07T12:24:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,322 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f91da6f9b6444bcad17721139edefbc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730982285492 2024-11-07T12:24:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,323 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3e4d73c7cf48e882bdd5775e622f7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,341 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:47,342 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/bedfa061c8524631afce00343ff2949b is 50, key is test_row_0/C:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075679f9e7d8304d62b38f80258a956d71_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_1/A:col10/1730982287266/Put/seqid=0 2024-11-07T12:24:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742208_1384 (size=12493) 2024-11-07T12:24:47,407 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/bedfa061c8524631afce00343ff2949b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/bedfa061c8524631afce00343ff2949b 2024-11-07T12:24:47,413 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into bedfa061c8524631afce00343ff2949b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:47,413 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:47,413 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982287220; duration=0sec 2024-11-07T12:24:47,413 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:47,413 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742209_1385 (size=9814) 2024-11-07T12:24:47,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:47,431 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075679f9e7d8304d62b38f80258a956d71_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075679f9e7d8304d62b38f80258a956d71_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:47,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/da5987c9ce29412db4258ca0a87bee8e, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:47,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/da5987c9ce29412db4258ca0a87bee8e is 175, key is test_row_1/A:col10/1730982287266/Put/seqid=0 2024-11-07T12:24:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742210_1386 (size=22461) 2024-11-07T12:24:47,453 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/da5987c9ce29412db4258ca0a87bee8e 2024-11-07T12:24:47,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/843b20c8267e4cd8a0591412e7ca712b is 50, key is test_row_1/B:col10/1730982287266/Put/seqid=0 2024-11-07T12:24:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:24:47,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742211_1387 (size=9757) 2024-11-07T12:24:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982347642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:47,720 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#318 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:47,720 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/3c5ab8ad441144a18ac470d235c4f9de is 175, key is test_row_0/A:col10/1730982286923/Put/seqid=0 2024-11-07T12:24:47,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742212_1388 (size=31447) 2024-11-07T12:24:47,757 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/3c5ab8ad441144a18ac470d235c4f9de as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de 2024-11-07T12:24:47,764 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into 3c5ab8ad441144a18ac470d235c4f9de(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:47,764 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:47,764 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982287220; duration=0sec 2024-11-07T12:24:47,765 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:47,765 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982347762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:24:47,884 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/843b20c8267e4cd8a0591412e7ca712b 2024-11-07T12:24:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/9c150972314c4ebbb7cb9e72ee6ac948 is 50, key is test_row_1/C:col10/1730982287266/Put/seqid=0 2024-11-07T12:24:47,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742213_1389 (size=9757) 2024-11-07T12:24:47,913 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/9c150972314c4ebbb7cb9e72ee6ac948 2024-11-07T12:24:47,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/da5987c9ce29412db4258ca0a87bee8e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e 2024-11-07T12:24:47,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e, entries=100, sequenceid=169, filesize=21.9 K 2024-11-07T12:24:47,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/843b20c8267e4cd8a0591412e7ca712b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b 2024-11-07T12:24:47,927 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b, entries=100, sequenceid=169, filesize=9.5 K 2024-11-07T12:24:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/9c150972314c4ebbb7cb9e72ee6ac948 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948 2024-11-07T12:24:47,933 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948, entries=100, sequenceid=169, filesize=9.5 K 2024-11-07T12:24:47,934 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e34bd2129ac88e7686bed4817db84e03 in 627ms, sequenceid=169, compaction requested=false 2024-11-07T12:24:47,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:47,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:47,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-07T12:24:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-07T12:24:47,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-07T12:24:47,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 783 msec 2024-11-07T12:24:47,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 788 msec 2024-11-07T12:24:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:47,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:47,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:24:47,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cfbcc86bd0ff40bd8aca871c5a3d9196_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:47,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742214_1390 (size=14794) 2024-11-07T12:24:47,996 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:48,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cfbcc86bd0ff40bd8aca871c5a3d9196_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfbcc86bd0ff40bd8aca871c5a3d9196_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:48,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/db9ef5d703924a88bb2d369bb2b84910, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:48,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/db9ef5d703924a88bb2d369bb2b84910 is 175, key is test_row_0/A:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:48,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742215_1391 (size=39749) 2024-11-07T12:24:48,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982348019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:48,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982348130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T12:24:48,291 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:48,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T12:24:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:48,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:48,294 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:48,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982348341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:48,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/db9ef5d703924a88bb2d369bb2b84910 2024-11-07T12:24:48,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/920a756841ce487aa90a8398dfb9f23e is 50, key is test_row_0/B:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:48,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742216_1392 (size=12151) 2024-11-07T12:24:48,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:24:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:24:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982348651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:48,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:24:48,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:48,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/920a756841ce487aa90a8398dfb9f23e 2024-11-07T12:24:48,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3a4eba89d1d14938a79bba57780cc09c is 50, key is test_row_0/C:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:48,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742217_1393 (size=12151) 2024-11-07T12:24:48,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3a4eba89d1d14938a79bba57780cc09c 2024-11-07T12:24:48,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/db9ef5d703924a88bb2d369bb2b84910 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910 2024-11-07T12:24:48,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910, entries=200, sequenceid=199, filesize=38.8 K 2024-11-07T12:24:48,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/920a756841ce487aa90a8398dfb9f23e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e 2024-11-07T12:24:48,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-07T12:24:48,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3a4eba89d1d14938a79bba57780cc09c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c 2024-11-07T12:24:48,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c, entries=150, sequenceid=199, filesize=11.9 K 2024-11-07T12:24:48,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e34bd2129ac88e7686bed4817db84e03 in 874ms, sequenceid=199, compaction requested=true 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:48,856 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:48,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:48,856 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:48,860 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:48,860 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:48,860 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:48,860 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:48,860 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,860 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,860 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=91.5 K 2024-11-07T12:24:48,860 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fa6d37a0349742ad9fbf0f601d7a4deb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=33.6 K 2024-11-07T12:24:48,860 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,860 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910] 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6d37a0349742ad9fbf0f601d7a4deb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5ab8ad441144a18ac470d235c4f9de, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 843b20c8267e4cd8a0591412e7ca712b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730982287123 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5987c9ce29412db4258ca0a87bee8e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730982287123 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 920a756841ce487aa90a8398dfb9f23e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:48,861 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9ef5d703924a88bb2d369bb2b84910, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:48,875 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:48,877 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:48,877 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107e086676e3faf4f0c8c7cdbbc49c6cd22_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:48,878 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/166186f6ae484ab9804bc0fe919290ce is 50, key is test_row_0/B:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:48,880 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107e086676e3faf4f0c8c7cdbbc49c6cd22_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:48,880 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e086676e3faf4f0c8c7cdbbc49c6cd22_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:48,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742218_1394 (size=12595) 2024-11-07T12:24:48,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742219_1395 (size=4469) 2024-11-07T12:24:48,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T12:24:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:48,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:24:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110786364601e2784b4599fcd5b90d3c1fa0_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982287996/Put/seqid=0 2024-11-07T12:24:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742220_1396 (size=12304) 2024-11-07T12:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:49,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:49,285 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:49,286 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/98f3091c2fca4beba6cd07382fae0d7b is 175, key is test_row_0/A:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:49,288 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/166186f6ae484ab9804bc0fe919290ce as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/166186f6ae484ab9804bc0fe919290ce 2024-11-07T12:24:49,293 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into 166186f6ae484ab9804bc0fe919290ce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:49,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:49,293 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982288856; duration=0sec 2024-11-07T12:24:49,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:49,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:49,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:49,294 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:49,294 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:49,294 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:49,294 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/bedfa061c8524631afce00343ff2949b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=33.6 K 2024-11-07T12:24:49,294 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bedfa061c8524631afce00343ff2949b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730982286923 2024-11-07T12:24:49,295 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c150972314c4ebbb7cb9e72ee6ac948, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1730982287123 2024-11-07T12:24:49,295 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4eba89d1d14938a79bba57780cc09c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:49,310 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:49,311 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/de47400b7d5f4805af442711f315b98d is 50, key is test_row_0/C:col10/1730982287536/Put/seqid=0 2024-11-07T12:24:49,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742221_1397 (size=31549) 2024-11-07T12:24:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742222_1398 (size=12595) 2024-11-07T12:24:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:49,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110786364601e2784b4599fcd5b90d3c1fa0_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786364601e2784b4599fcd5b90d3c1fa0_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/14004aeff8f64c7e9ed72800cb8ae8f4, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/14004aeff8f64c7e9ed72800cb8ae8f4 is 175, key is test_row_0/A:col10/1730982287996/Put/seqid=0 2024-11-07T12:24:49,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742223_1399 (size=31105) 2024-11-07T12:24:49,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982349346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:49,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982349461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982349671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:49,717 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/98f3091c2fca4beba6cd07382fae0d7b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b 2024-11-07T12:24:49,720 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into 98f3091c2fca4beba6cd07382fae0d7b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:49,720 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:49,720 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982288856; duration=0sec 2024-11-07T12:24:49,720 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:49,720 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:49,729 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/de47400b7d5f4805af442711f315b98d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/de47400b7d5f4805af442711f315b98d 2024-11-07T12:24:49,732 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into de47400b7d5f4805af442711f315b98d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:49,732 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:49,732 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982288856; duration=0sec 2024-11-07T12:24:49,733 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:49,733 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:49,740 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/14004aeff8f64c7e9ed72800cb8ae8f4 2024-11-07T12:24:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/e53f1a5bbce448aa890f0d35a4f2584a is 50, key is test_row_0/B:col10/1730982287996/Put/seqid=0 2024-11-07T12:24:49,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742224_1400 (size=12151) 2024-11-07T12:24:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982349990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/e53f1a5bbce448aa890f0d35a4f2584a 2024-11-07T12:24:50,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3f9b1f24ad30423c8a79fea77f721a15 is 50, key is test_row_0/C:col10/1730982287996/Put/seqid=0 2024-11-07T12:24:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742225_1401 (size=12151) 2024-11-07T12:24:50,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1730982350350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982350351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:50,357 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,357 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982350382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:50,385 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730982350383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:50,393 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:24:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:50,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982350501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:50,566 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3f9b1f24ad30423c8a79fea77f721a15 2024-11-07T12:24:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/14004aeff8f64c7e9ed72800cb8ae8f4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4 2024-11-07T12:24:50,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4, entries=150, sequenceid=208, filesize=30.4 K 2024-11-07T12:24:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/e53f1a5bbce448aa890f0d35a4f2584a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a 2024-11-07T12:24:50,576 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T12:24:50,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3f9b1f24ad30423c8a79fea77f721a15 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15 2024-11-07T12:24:50,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T12:24:50,580 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for e34bd2129ac88e7686bed4817db84e03 in 1671ms, sequenceid=208, compaction requested=false 2024-11-07T12:24:50,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:50,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:50,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T12:24:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T12:24:50,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T12:24:50,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2870 sec 2024-11-07T12:24:50,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.2930 sec 2024-11-07T12:24:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:51,517 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:51,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T12:24:51,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735f2799c7a4c4c15a83df2e3df551115_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:51,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742226_1402 (size=14794) 2024-11-07T12:24:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982351544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982351650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:51,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982351861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:51,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:51,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:51,930 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735f2799c7a4c4c15a83df2e3df551115_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735f2799c7a4c4c15a83df2e3df551115_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:51,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/492a9d935ecf4b95b7f24955e9fabb13, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:51,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/492a9d935ecf4b95b7f24955e9fabb13 is 175, key is test_row_0/A:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742227_1403 (size=39749) 2024-11-07T12:24:52,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982352171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:52,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/492a9d935ecf4b95b7f24955e9fabb13 2024-11-07T12:24:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6fb66ba91f4744d5991f323bddc1aa0e is 50, key is test_row_0/B:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:52,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742228_1404 (size=12151) 2024-11-07T12:24:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T12:24:52,461 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:52,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T12:24:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:52,464 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:52,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:52,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:52,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:24:52,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982352681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:52,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6fb66ba91f4744d5991f323bddc1aa0e 2024-11-07T12:24:52,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3bc2513306404b04a75c85d0f3d51baf is 50, key is test_row_0/C:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:52,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742229_1405 (size=12151) 2024-11-07T12:24:52,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:24:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:52,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:24:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:52,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:53,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:24:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:53,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3bc2513306404b04a75c85d0f3d51baf 2024-11-07T12:24:53,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/492a9d935ecf4b95b7f24955e9fabb13 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13 2024-11-07T12:24:53,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13, entries=200, sequenceid=239, filesize=38.8 K 2024-11-07T12:24:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6fb66ba91f4744d5991f323bddc1aa0e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e 2024-11-07T12:24:53,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:24:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3bc2513306404b04a75c85d0f3d51baf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf 2024-11-07T12:24:53,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf, entries=150, sequenceid=239, filesize=11.9 K 2024-11-07T12:24:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e34bd2129ac88e7686bed4817db84e03 in 1661ms, sequenceid=239, compaction requested=true 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:53,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:53,178 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:53,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:53,179 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:53,179 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:53,179 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:53,179 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:53,179 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,179 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,179 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/166186f6ae484ab9804bc0fe919290ce, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.0 K 2024-11-07T12:24:53,179 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=100.0 K 2024-11-07T12:24:53,179 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,179 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13] 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 166186f6ae484ab9804bc0fe919290ce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e53f1a5bbce448aa890f0d35a4f2584a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982287996 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f3091c2fca4beba6cd07382fae0d7b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fb66ba91f4744d5991f323bddc1aa0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289328 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14004aeff8f64c7e9ed72800cb8ae8f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982287996 2024-11-07T12:24:53,180 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492a9d935ecf4b95b7f24955e9fabb13, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289298 2024-11-07T12:24:53,187 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:53,187 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/a78d6a515d8a428a9d25af8b3183de04 is 50, key is test_row_0/B:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:53,188 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:53,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742230_1406 (size=12697) 2024-11-07T12:24:53,190 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070bdcc7c17bf74c0baa87e8ca0d4ba25b_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:53,192 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070bdcc7c17bf74c0baa87e8ca0d4ba25b_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:53,193 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070bdcc7c17bf74c0baa87e8ca0d4ba25b_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:53,196 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/a78d6a515d8a428a9d25af8b3183de04 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a78d6a515d8a428a9d25af8b3183de04 2024-11-07T12:24:53,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742231_1407 (size=4469) 2024-11-07T12:24:53,202 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into a78d6a515d8a428a9d25af8b3183de04(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:53,202 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:53,202 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982293178; duration=0sec 2024-11-07T12:24:53,202 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:53,202 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:53,202 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:53,203 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:53,203 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:53,203 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,203 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/de47400b7d5f4805af442711f315b98d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.0 K 2024-11-07T12:24:53,203 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting de47400b7d5f4805af442711f315b98d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1730982287536 2024-11-07T12:24:53,204 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f9b1f24ad30423c8a79fea77f721a15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730982287996 2024-11-07T12:24:53,204 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc2513306404b04a75c85d0f3d51baf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289328 2024-11-07T12:24:53,211 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:53,212 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/51e94371475d45bdba4b94e1b22827dd is 50, key is test_row_0/C:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:53,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742232_1408 (size=12697) 2024-11-07T12:24:53,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T12:24:53,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:53,229 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:24:53,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071f152f1c707e432aa10989ca8989bb70_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982291542/Put/seqid=0 2024-11-07T12:24:53,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742233_1409 (size=12304) 2024-11-07T12:24:53,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:53,241 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071f152f1c707e432aa10989ca8989bb70_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071f152f1c707e432aa10989ca8989bb70_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/cd7469d4123c476889a6f50feac89cec, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/cd7469d4123c476889a6f50feac89cec is 175, key is test_row_0/A:col10/1730982291542/Put/seqid=0 2024-11-07T12:24:53,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742234_1410 (size=31105) 2024-11-07T12:24:53,600 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:53,600 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/b2a0aa91e5b747e99eb51837da84ab0e is 175, key is test_row_0/A:col10/1730982291516/Put/seqid=0 2024-11-07T12:24:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:53,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742235_1411 (size=31651) 2024-11-07T12:24:53,623 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/b2a0aa91e5b747e99eb51837da84ab0e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e 2024-11-07T12:24:53,625 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/51e94371475d45bdba4b94e1b22827dd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/51e94371475d45bdba4b94e1b22827dd 2024-11-07T12:24:53,629 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into b2a0aa91e5b747e99eb51837da84ab0e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:53,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:53,630 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982293178; duration=0sec 2024-11-07T12:24:53,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:53,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:53,636 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into 51e94371475d45bdba4b94e1b22827dd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:53,636 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:53,636 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982293178; duration=0sec 2024-11-07T12:24:53,636 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:53,636 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:53,648 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/cd7469d4123c476889a6f50feac89cec 2024-11-07T12:24:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/b7a40678765c4807a02cfdca54019b68 is 50, key is test_row_0/B:col10/1730982291542/Put/seqid=0 2024-11-07T12:24:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:53,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:53,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742236_1412 (size=12151) 2024-11-07T12:24:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982353904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982354020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/b7a40678765c4807a02cfdca54019b68 2024-11-07T12:24:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a244b088b43f48bd8f2fa2fd0579f3a9 is 50, key is test_row_0/C:col10/1730982291542/Put/seqid=0 2024-11-07T12:24:54,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742237_1413 (size=12151) 2024-11-07T12:24:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982354231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,513 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a244b088b43f48bd8f2fa2fd0579f3a9 2024-11-07T12:24:54,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/cd7469d4123c476889a6f50feac89cec as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec 2024-11-07T12:24:54,521 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec, entries=150, sequenceid=248, filesize=30.4 K 2024-11-07T12:24:54,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/b7a40678765c4807a02cfdca54019b68 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68 2024-11-07T12:24:54,531 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T12:24:54,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a244b088b43f48bd8f2fa2fd0579f3a9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9 2024-11-07T12:24:54,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T12:24:54,538 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for e34bd2129ac88e7686bed4817db84e03 in 1308ms, sequenceid=248, compaction requested=false 2024-11-07T12:24:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T12:24:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T12:24:54,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T12:24:54,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0740 sec 2024-11-07T12:24:54,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0800 sec 2024-11-07T12:24:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:54,546 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:54,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T12:24:54,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110712d2f96988da48debbafabfbf77c1a6d_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742238_1414 (size=14994) 2024-11-07T12:24:54,566 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:54,570 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110712d2f96988da48debbafabfbf77c1a6d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712d2f96988da48debbafabfbf77c1a6d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:54,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/dbea13e670b14ddab14bc6b07c8573bd, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:54,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/dbea13e670b14ddab14bc6b07c8573bd is 175, key is test_row_0/A:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:54,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742239_1415 (size=39949) 2024-11-07T12:24:54,577 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/dbea13e670b14ddab14bc6b07c8573bd 2024-11-07T12:24:54,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/4d39b62edaf74d2285bdc6fc6d9bbdab is 50, key is test_row_0/B:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:54,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742240_1416 (size=12301) 2024-11-07T12:24:54,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982354583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T12:24:54,611 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:54,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T12:24:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:54,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:54,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:54,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982354701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:54,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:54,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982354911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:54,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:54,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:54,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:54,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/4d39b62edaf74d2285bdc6fc6d9bbdab 2024-11-07T12:24:54,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/60962122613e4dfcb25b7bd3b1e74af7 is 50, key is test_row_0/C:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742241_1417 (size=12301) 2024-11-07T12:24:55,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982355220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:55,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:55,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:55,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:55,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:55,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/60962122613e4dfcb25b7bd3b1e74af7 2024-11-07T12:24:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/dbea13e670b14ddab14bc6b07c8573bd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd 2024-11-07T12:24:55,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd, entries=200, sequenceid=279, filesize=39.0 K 2024-11-07T12:24:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/4d39b62edaf74d2285bdc6fc6d9bbdab as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab 2024-11-07T12:24:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:24:55,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/60962122613e4dfcb25b7bd3b1e74af7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7 2024-11-07T12:24:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7, entries=150, sequenceid=279, filesize=12.0 K 2024-11-07T12:24:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for e34bd2129ac88e7686bed4817db84e03 in 863ms, sequenceid=279, compaction requested=true 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:55,410 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:55,410 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:55,411 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,411 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,411 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=100.3 K 2024-11-07T12:24:55,411 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a78d6a515d8a428a9d25af8b3183de04, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.3 K 2024-11-07T12:24:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,411 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd] 2024-11-07T12:24:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a78d6a515d8a428a9d25af8b3183de04, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289328 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2a0aa91e5b747e99eb51837da84ab0e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289328 2024-11-07T12:24:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd7469d4123c476889a6f50feac89cec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982291518 2024-11-07T12:24:55,411 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a40678765c4807a02cfdca54019b68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982291518 2024-11-07T12:24:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,412 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbea13e670b14ddab14bc6b07c8573bd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293837 2024-11-07T12:24:55,412 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d39b62edaf74d2285bdc6fc6d9bbdab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293888 2024-11-07T12:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/489de692e65c4cf3bcf0c9f7b467b609 is 50, key is test_row_0/B:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,419 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b2660f90e13f4c2e8f9b8c682c6a8a3f_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,438 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b2660f90e13f4c2e8f9b8c682c6a8a3f_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b2660f90e13f4c2e8f9b8c682c6a8a3f_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742242_1418 (size=12949) 2024-11-07T12:24:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742243_1419 (size=4469) 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,447 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#346 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,447 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/ec2a5cf6d5be45689d35d4eae43e8494 is 175, key is test_row_0/A:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742244_1420 (size=31903) 2024-11-07T12:24:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T12:24:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T12:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070cb9746d441f4beabf8e1e660027834d_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982294571/Put/seqid=0 2024-11-07T12:24:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742245_1421 (size=9914) 2024-11-07T12:24:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,571 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070cb9746d441f4beabf8e1e660027834d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070cb9746d441f4beabf8e1e660027834d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d9f57c6803f541fcaebfdf105ccde62b, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d9f57c6803f541fcaebfdf105ccde62b is 175, key is test_row_0/A:col10/1730982294571/Put/seqid=0 2024-11-07T12:24:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742246_1422 (size=22561) 2024-11-07T12:24:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,581 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d9f57c6803f541fcaebfdf105ccde62b 2024-11-07T12:24:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/723c7db118ad4d138daa3171cebf915a is 50, key is test_row_0/B:col10/1730982294571/Put/seqid=0 2024-11-07T12:24:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742247_1423 (size=9857) 2024-11-07T12:24:55,614 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/723c7db118ad4d138daa3171cebf915a 2024-11-07T12:24:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/afda486175b24a90a84612ace5cdeddd is 50, key is test_row_0/C:col10/1730982294571/Put/seqid=0 2024-11-07T12:24:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742248_1424 (size=9857) 2024-11-07T12:24:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/489de692e65c4cf3bcf0c9f7b467b609 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/489de692e65c4cf3bcf0c9f7b467b609 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into 489de692e65c4cf3bcf0c9f7b467b609(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:55,851 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:55,851 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982295410; duration=0sec 2024-11-07T12:24:55,851 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:55,851 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,851 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,854 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:55,854 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:55,854 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,854 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/51e94371475d45bdba4b94e1b22827dd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.3 K 2024-11-07T12:24:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,857 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e94371475d45bdba4b94e1b22827dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1730982289328 2024-11-07T12:24:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,858 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/ec2a5cf6d5be45689d35d4eae43e8494 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494 2024-11-07T12:24:55,858 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a244b088b43f48bd8f2fa2fd0579f3a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730982291518 2024-11-07T12:24:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,858 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 60962122613e4dfcb25b7bd3b1e74af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293888 2024-11-07T12:24:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,863 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into ec2a5cf6d5be45689d35d4eae43e8494(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:55,863 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,863 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982295410; duration=0sec 2024-11-07T12:24:55,863 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:55,863 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,870 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:55,871 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd is 50, key is test_row_0/C:col10/1730982294545/Put/seqid=0 2024-11-07T12:24:55,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742249_1425 (size=12949) 2024-11-07T12:24:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/afda486175b24a90a84612ace5cdeddd 2024-11-07T12:24:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d9f57c6803f541fcaebfdf105ccde62b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b 2024-11-07T12:24:56,047 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b, entries=100, sequenceid=286, filesize=22.0 K 2024-11-07T12:24:56,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/723c7db118ad4d138daa3171cebf915a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a 2024-11-07T12:24:56,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a, entries=100, sequenceid=286, filesize=9.6 K 2024-11-07T12:24:56,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/afda486175b24a90a84612ace5cdeddd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd 2024-11-07T12:24:56,054 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd, entries=100, sequenceid=286, filesize=9.6 K 2024-11-07T12:24:56,055 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=67.09 KB/68700 for e34bd2129ac88e7686bed4817db84e03 in 518ms, sequenceid=286, compaction requested=false 2024-11-07T12:24:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:56,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T12:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T12:24:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:56,062 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:56,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:24:56,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T12:24:56,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-07T12:24:56,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4510 sec 2024-11-07T12:24:56,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076bc4fa804b72410c8f7b1a8822a3c43f_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:56,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742250_1426 (size=14994) 2024-11-07T12:24:56,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982356197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982356311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:56,322 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd 2024-11-07T12:24:56,325 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into 3f7a3c19cb4a47a4a94dc3a3ba4beedd(size=12.6 K), total size for store is 22.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:56,325 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:56,325 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982295410; duration=0sec 2024-11-07T12:24:56,325 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:56,325 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:56,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:56,476 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076bc4fa804b72410c8f7b1a8822a3c43f_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bc4fa804b72410c8f7b1a8822a3c43f_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:56,476 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/34051de0645f4453b558b53532052214, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:56,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/34051de0645f4453b558b53532052214 is 175, key is test_row_0/A:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:56,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742251_1427 (size=39949) 2024-11-07T12:24:56,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982356530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T12:24:56,761 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:24:56,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:24:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T12:24:56,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:24:56,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:24:56,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:24:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:56,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982356841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:56,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:56,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/34051de0645f4453b558b53532052214 2024-11-07T12:24:56,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/9fe2688a9e844a48bb0f7366be0c4d18 is 50, key is test_row_0/B:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:56,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742252_1428 (size=12301) 2024-11-07T12:24:56,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:56,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:57,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/9fe2688a9e844a48bb0f7366be0c4d18 2024-11-07T12:24:57,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b7d1387fa48e45a6bae3464729126f51 is 50, key is test_row_0/C:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:57,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742253_1429 (size=12301) 2024-11-07T12:24:57,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982357361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:57,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:57,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:57,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:24:57,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b7d1387fa48e45a6bae3464729126f51 2024-11-07T12:24:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/34051de0645f4453b558b53532052214 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214 2024-11-07T12:24:57,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214, entries=200, sequenceid=302, filesize=39.0 K 2024-11-07T12:24:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/9fe2688a9e844a48bb0f7366be0c4d18 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18 2024-11-07T12:24:57,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18, entries=150, sequenceid=302, filesize=12.0 K 2024-11-07T12:24:57,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/b7d1387fa48e45a6bae3464729126f51 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51 2024-11-07T12:24:57,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51, entries=150, sequenceid=302, filesize=12.0 K 2024-11-07T12:24:57,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e34bd2129ac88e7686bed4817db84e03 in 1669ms, sequenceid=302, compaction requested=true 2024-11-07T12:24:57,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:57,732 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:57,732 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:24:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:57,734 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:57,734 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:24:57,734 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,734 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=92.2 K 2024-11-07T12:24:57,734 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,734 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214] 2024-11-07T12:24:57,735 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:57,735 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec2a5cf6d5be45689d35d4eae43e8494, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293888 2024-11-07T12:24:57,735 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:24:57,735 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,735 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/489de692e65c4cf3bcf0c9f7b467b609, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=34.3 K 2024-11-07T12:24:57,735 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f57c6803f541fcaebfdf105ccde62b, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982294571 2024-11-07T12:24:57,736 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34051de0645f4453b558b53532052214, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982295961 2024-11-07T12:24:57,736 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 489de692e65c4cf3bcf0c9f7b467b609, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293888 2024-11-07T12:24:57,736 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 723c7db118ad4d138daa3171cebf915a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982294571 2024-11-07T12:24:57,736 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe2688a9e844a48bb0f7366be0c4d18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982296040 2024-11-07T12:24:57,746 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:57,747 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:57,748 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110761fe04fefa554bd9901fe7ab51fc18ae_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:57,748 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4 is 50, key is test_row_0/B:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:57,750 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110761fe04fefa554bd9901fe7ab51fc18ae_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:57,750 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110761fe04fefa554bd9901fe7ab51fc18ae_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:57,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742254_1430 (size=13051) 2024-11-07T12:24:57,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742255_1431 (size=4469) 2024-11-07T12:24:57,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T12:24:57,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:57,831 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:24:57,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110738350da9f4c44aaba5193713f38912b5_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982296196/Put/seqid=0 2024-11-07T12:24:57,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742256_1432 (size=12454) 2024-11-07T12:24:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:58,156 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:58,157 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d8afe5acae56450c86f9ccc72953c178 is 175, key is test_row_0/A:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:58,162 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4 2024-11-07T12:24:58,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742257_1433 (size=32005) 2024-11-07T12:24:58,169 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into fc3fabe5ecac4f5b8a241cb2ee5e5ac4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:58,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:58,169 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982297732; duration=0sec 2024-11-07T12:24:58,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:24:58,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:24:58,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:24:58,170 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:24:58,170 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:24:58,170 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:58,171 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=34.3 K 2024-11-07T12:24:58,171 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7a3c19cb4a47a4a94dc3a3ba4beedd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1730982293888 2024-11-07T12:24:58,172 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting afda486175b24a90a84612ace5cdeddd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982294571 2024-11-07T12:24:58,172 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d1387fa48e45a6bae3464729126f51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982296040 2024-11-07T12:24:58,190 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:24:58,191 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/4c4adc657e7749f0bc85f16a6913892d is 50, key is test_row_0/C:col10/1730982296046/Put/seqid=0 2024-11-07T12:24:58,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742258_1434 (size=13051) 2024-11-07T12:24:58,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:58,243 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110738350da9f4c44aaba5193713f38912b5_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738350da9f4c44aaba5193713f38912b5_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/8dbcb99cfb664b2aa774487ce1de7ec7, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:58,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/8dbcb99cfb664b2aa774487ce1de7ec7 is 175, key is test_row_0/A:col10/1730982296196/Put/seqid=0 2024-11-07T12:24:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742259_1435 (size=31255) 2024-11-07T12:24:58,249 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/8dbcb99cfb664b2aa774487ce1de7ec7 2024-11-07T12:24:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/909bc58c7df34a6abd259ae8c43b3356 is 50, key is test_row_0/B:col10/1730982296196/Put/seqid=0 2024-11-07T12:24:58,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742260_1436 (size=12301) 2024-11-07T12:24:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:58,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. as already flushing 2024-11-07T12:24:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982358476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:58,571 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/d8afe5acae56450c86f9ccc72953c178 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178 2024-11-07T12:24:58,574 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into d8afe5acae56450c86f9ccc72953c178(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:58,574 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:58,574 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982297731; duration=0sec 2024-11-07T12:24:58,574 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:58,574 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:24:58,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982358581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:58,599 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/4c4adc657e7749f0bc85f16a6913892d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/4c4adc657e7749f0bc85f16a6913892d 2024-11-07T12:24:58,606 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into 4c4adc657e7749f0bc85f16a6913892d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:24:58,606 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:58,606 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982297732; duration=0sec 2024-11-07T12:24:58,606 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:24:58,606 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:24:58,662 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/909bc58c7df34a6abd259ae8c43b3356 2024-11-07T12:24:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/823c694d900746ceb93e4e5e19898756 is 50, key is test_row_0/C:col10/1730982296196/Put/seqid=0 2024-11-07T12:24:58,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742261_1437 (size=12301) 2024-11-07T12:24:58,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982358800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:24:59,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/823c694d900746ceb93e4e5e19898756 2024-11-07T12:24:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/8dbcb99cfb664b2aa774487ce1de7ec7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7 2024-11-07T12:24:59,079 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7, entries=150, sequenceid=325, filesize=30.5 K 2024-11-07T12:24:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/909bc58c7df34a6abd259ae8c43b3356 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356 2024-11-07T12:24:59,083 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T12:24:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/823c694d900746ceb93e4e5e19898756 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756 2024-11-07T12:24:59,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T12:24:59,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e34bd2129ac88e7686bed4817db84e03 in 1256ms, sequenceid=325, compaction requested=false 2024-11-07T12:24:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:24:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:24:59,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T12:24:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T12:24:59,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T12:24:59,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-11-07T12:24:59,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3300 sec 2024-11-07T12:24:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:59,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:24:59,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:24:59,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074c602ac41de24f908d26b0336a1c32fb_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982298461/Put/seqid=0 2024-11-07T12:24:59,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742262_1438 (size=14994) 2024-11-07T12:24:59,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982359226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:59,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982359341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:59,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:24:59,527 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074c602ac41de24f908d26b0336a1c32fb_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074c602ac41de24f908d26b0336a1c32fb_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:24:59,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/e67056da2c394b10b4bc700b145224f7, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:24:59,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/e67056da2c394b10b4bc700b145224f7 is 175, key is test_row_0/A:col10/1730982298461/Put/seqid=0 2024-11-07T12:24:59,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742263_1439 (size=39949) 2024-11-07T12:24:59,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/e67056da2c394b10b4bc700b145224f7 2024-11-07T12:24:59,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982359562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:59,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/8f3278100c664a0e835b7849a5cd9732 is 50, key is test_row_0/B:col10/1730982298461/Put/seqid=0 2024-11-07T12:24:59,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742264_1440 (size=12301) 2024-11-07T12:24:59,715 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-07T12:24:59,715 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:59,715 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,715 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:59,715 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-07T12:24:59,715 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:59,715 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,715 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:59,721 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-07T12:24:59,721 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:59,721 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,721 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:59,725 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-07T12:24:59,725 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:59,725 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,725 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:59,726 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-07T12:24:59,726 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:24:59,726 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,726 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:24:59,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:24:59,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:24:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982359881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:24:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:24:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:24:59,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/8f3278100c664a0e835b7849a5cd9732 2024-11-07T12:25:00,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a593d6d949cf46f389cdeb4c1a30ff5b is 50, key is test_row_0/C:col10/1730982298461/Put/seqid=0 2024-11-07T12:25:00,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742265_1441 (size=12301) 2024-11-07T12:25:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42844 deadline: 1730982360391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:00,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42858 deadline: 1730982360401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:00,402 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:00,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a593d6d949cf46f389cdeb4c1a30ff5b 2024-11-07T12:25:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42822 deadline: 1730982360411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:00,411 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in e34bd2129ac88e7686bed4817db84e03 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:00,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/e67056da2c394b10b4bc700b145224f7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7 2024-11-07T12:25:00,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7, entries=200, sequenceid=343, filesize=39.0 K 2024-11-07T12:25:00,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/8f3278100c664a0e835b7849a5cd9732 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732 2024-11-07T12:25:00,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T12:25:00,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/a593d6d949cf46f389cdeb4c1a30ff5b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b 2024-11-07T12:25:00,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T12:25:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e34bd2129ac88e7686bed4817db84e03 in 1312ms, sequenceid=343, compaction requested=true 2024-11-07T12:25:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:25:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e34bd2129ac88e7686bed4817db84e03:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/A is initiating minor compaction (all files) 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/B is initiating minor compaction (all files) 2024-11-07T12:25:00,426 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/A in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:00,426 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/B in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:00,426 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=100.8 K 2024-11-07T12:25:00,426 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.8 K 2024-11-07T12:25:00,426 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:00,426 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7] 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3fabe5ecac4f5b8a241cb2ee5e5ac4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982296040 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8afe5acae56450c86f9ccc72953c178, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982296040 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 909bc58c7df34a6abd259ae8c43b3356, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730982296174 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dbcb99cfb664b2aa774487ce1de7ec7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730982296174 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f3278100c664a0e835b7849a5cd9732, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730982298432 2024-11-07T12:25:00,427 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e67056da2c394b10b4bc700b145224f7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730982298432 2024-11-07T12:25:00,431 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:00,432 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107035fc7ed70c943699d11ce33f12cd910_e34bd2129ac88e7686bed4817db84e03 store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:00,433 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:00,433 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6c4d013cf42947dfa34ac9f30d49f9e7 is 50, key is test_row_0/B:col10/1730982298461/Put/seqid=0 2024-11-07T12:25:00,435 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107035fc7ed70c943699d11ce33f12cd910_e34bd2129ac88e7686bed4817db84e03, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:00,436 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107035fc7ed70c943699d11ce33f12cd910_e34bd2129ac88e7686bed4817db84e03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:00,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742266_1442 (size=13153) 2024-11-07T12:25:00,441 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/6c4d013cf42947dfa34ac9f30d49f9e7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c4d013cf42947dfa34ac9f30d49f9e7 2024-11-07T12:25:00,444 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/B of e34bd2129ac88e7686bed4817db84e03 into 6c4d013cf42947dfa34ac9f30d49f9e7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:00,444 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:25:00,445 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/B, priority=13, startTime=1730982300426; duration=0sec 2024-11-07T12:25:00,445 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:00,445 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:B 2024-11-07T12:25:00,445 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:00,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742267_1443 (size=4469) 2024-11-07T12:25:00,446 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:00,446 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#A#compaction#364 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:00,446 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): e34bd2129ac88e7686bed4817db84e03/C is initiating minor compaction (all files) 2024-11-07T12:25:00,446 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e34bd2129ac88e7686bed4817db84e03/C in TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:00,446 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/4c4adc657e7749f0bc85f16a6913892d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp, totalSize=36.8 K 2024-11-07T12:25:00,446 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/03a9625c5a56479fa69c72167a776698 is 175, key is test_row_0/A:col10/1730982298461/Put/seqid=0 2024-11-07T12:25:00,447 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4adc657e7749f0bc85f16a6913892d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1730982296040 2024-11-07T12:25:00,448 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 823c694d900746ceb93e4e5e19898756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730982296174 2024-11-07T12:25:00,448 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a593d6d949cf46f389cdeb4c1a30ff5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730982298432 2024-11-07T12:25:00,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742268_1444 (size=32107) 2024-11-07T12:25:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:00,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:00,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:25:00,453 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-07T12:25:00,453 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:00,453 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:00,454 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:00,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:00,455 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/03a9625c5a56479fa69c72167a776698 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/03a9625c5a56479fa69c72167a776698 2024-11-07T12:25:00,456 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e34bd2129ac88e7686bed4817db84e03#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:00,457 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3e1204f6420d4105b3e9d9ba9f8b1d27 is 50, key is test_row_0/C:col10/1730982298461/Put/seqid=0 2024-11-07T12:25:00,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cc2ee6c593f749e88cba645ebdab21e7_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982300452/Put/seqid=0 2024-11-07T12:25:00,462 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-07T12:25:00,462 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:00,462 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:00,462 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:00,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:00,463 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/A of e34bd2129ac88e7686bed4817db84e03 into 03a9625c5a56479fa69c72167a776698(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:00,463 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:25:00,463 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/A, priority=13, startTime=1730982300425; duration=0sec 2024-11-07T12:25:00,463 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:00,463 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:A 2024-11-07T12:25:00,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742269_1445 (size=13153) 2024-11-07T12:25:00,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742270_1446 (size=12454) 2024-11-07T12:25:00,472 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:00,475 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cc2ee6c593f749e88cba645ebdab21e7_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc2ee6c593f749e88cba645ebdab21e7_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:00,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5102f6169ee2462782135e7392599bde, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:00,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5102f6169ee2462782135e7392599bde is 175, key is test_row_0/A:col10/1730982300452/Put/seqid=0 2024-11-07T12:25:00,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742271_1447 (size=31255) 2024-11-07T12:25:00,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5102f6169ee2462782135e7392599bde 2024-11-07T12:25:00,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:25:00,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2e1831246db94f8e8ea716185992cef4 is 50, key is test_row_0/B:col10/1730982300452/Put/seqid=0 2024-11-07T12:25:00,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742272_1448 (size=12301) 2024-11-07T12:25:00,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2e1831246db94f8e8ea716185992cef4 2024-11-07T12:25:00,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/40839fe2cf7e42159cfd741de38622c5 is 50, key is test_row_0/C:col10/1730982300452/Put/seqid=0 2024-11-07T12:25:00,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742273_1449 (size=12301) 2024-11-07T12:25:00,876 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/3e1204f6420d4105b3e9d9ba9f8b1d27 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3e1204f6420d4105b3e9d9ba9f8b1d27 2024-11-07T12:25:00,881 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e34bd2129ac88e7686bed4817db84e03/C of e34bd2129ac88e7686bed4817db84e03 into 3e1204f6420d4105b3e9d9ba9f8b1d27(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:00,881 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:25:00,881 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., storeName=e34bd2129ac88e7686bed4817db84e03/C, priority=13, startTime=1730982300426; duration=0sec 2024-11-07T12:25:00,881 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:00,881 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e34bd2129ac88e7686bed4817db84e03:C 2024-11-07T12:25:00,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/40839fe2cf7e42159cfd741de38622c5 2024-11-07T12:25:00,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/5102f6169ee2462782135e7392599bde as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5102f6169ee2462782135e7392599bde 2024-11-07T12:25:00,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5102f6169ee2462782135e7392599bde, entries=150, sequenceid=366, filesize=30.5 K 2024-11-07T12:25:00,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/2e1831246db94f8e8ea716185992cef4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e1831246db94f8e8ea716185992cef4 2024-11-07T12:25:00,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e1831246db94f8e8ea716185992cef4, entries=150, sequenceid=366, filesize=12.0 K 2024-11-07T12:25:00,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/40839fe2cf7e42159cfd741de38622c5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/40839fe2cf7e42159cfd741de38622c5 2024-11-07T12:25:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T12:25:00,932 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:00,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/40839fe2cf7e42159cfd741de38622c5, entries=150, sequenceid=366, filesize=12.0 K 2024-11-07T12:25:00,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=6.71 KB/6870 for e34bd2129ac88e7686bed4817db84e03 in 482ms, sequenceid=366, compaction requested=false 2024-11-07T12:25:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e34bd2129ac88e7686bed4817db84e03: 2024-11-07T12:25:01,402 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-07T12:25:01,402 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:01,402 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:01,402 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:01,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:05,265 WARN [regionserver/8944cbb06cd2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-07T12:25:05,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7] to archive 2024-11-07T12:25:05,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:05,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/103162e6e1c4449699c7ad51d3536f9e 2024-11-07T12:25:05,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/9e6cb0d906784bf68ebcbc243e52f4f3 2024-11-07T12:25:05,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/f170c4f56c0a45db8cf93768384cd161 2024-11-07T12:25:05,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/831aad9e74104526acfe32fe648735d9 2024-11-07T12:25:05,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5579467f305248f68465b8ba1dcec59f 2024-11-07T12:25:05,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/361abc196bbb48c294c1ec6f5402019c 2024-11-07T12:25:05,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/4409c5b6cdc041a7a904494ac979f36c 2024-11-07T12:25:05,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/0cefa69e34d24fe1a3213bd8e9393755 2024-11-07T12:25:05,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d48e77c2c78c4996986e0fb8214c3715 2024-11-07T12:25:05,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d0048aa44492412a8e6bc181e1c4a205 2024-11-07T12:25:05,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/3c5ab8ad441144a18ac470d235c4f9de 2024-11-07T12:25:05,278 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/da5987c9ce29412db4258ca0a87bee8e 2024-11-07T12:25:05,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/db9ef5d703924a88bb2d369bb2b84910 2024-11-07T12:25:05,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/98f3091c2fca4beba6cd07382fae0d7b 2024-11-07T12:25:05,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/14004aeff8f64c7e9ed72800cb8ae8f4 2024-11-07T12:25:05,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/492a9d935ecf4b95b7f24955e9fabb13 2024-11-07T12:25:05,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/b2a0aa91e5b747e99eb51837da84ab0e 2024-11-07T12:25:05,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/cd7469d4123c476889a6f50feac89cec 2024-11-07T12:25:05,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/dbea13e670b14ddab14bc6b07c8573bd 2024-11-07T12:25:05,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/ec2a5cf6d5be45689d35d4eae43e8494 2024-11-07T12:25:05,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d9f57c6803f541fcaebfdf105ccde62b 2024-11-07T12:25:05,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/34051de0645f4453b558b53532052214 2024-11-07T12:25:05,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/d8afe5acae56450c86f9ccc72953c178 2024-11-07T12:25:05,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/8dbcb99cfb664b2aa774487ce1de7ec7 2024-11-07T12:25:05,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/e67056da2c394b10b4bc700b145224f7 2024-11-07T12:25:05,292 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e72fddea58e4a16826cf2f0566a4c95, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/c83b0f333205416ea76ccc42126fed3c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fa6d37a0349742ad9fbf0f601d7a4deb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/166186f6ae484ab9804bc0fe919290ce, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a78d6a515d8a428a9d25af8b3183de04, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/489de692e65c4cf3bcf0c9f7b467b609, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732] to archive 2024-11-07T12:25:05,293 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:05,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0e20f633763147dfbb13677eb9e329c4 2024-11-07T12:25:05,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a90a330b694145e79b9661044d52675a 2024-11-07T12:25:05,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e72fddea58e4a16826cf2f0566a4c95 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e72fddea58e4a16826cf2f0566a4c95 2024-11-07T12:25:05,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/1eca09832adb47bd8510a093a3dbcf74 2024-11-07T12:25:05,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c09406974bb4306a5982f2e45173587 2024-11-07T12:25:05,298 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/0f81d7b2c20548edbdcf8beb60d02583 2024-11-07T12:25:05,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/c83b0f333205416ea76ccc42126fed3c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/c83b0f333205416ea76ccc42126fed3c 2024-11-07T12:25:05,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/85f29e7765044c3aa75628ae165c3031 2024-11-07T12:25:05,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/647b4a486e99424eaf6e357b1f3f309c 2024-11-07T12:25:05,301 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fa6d37a0349742ad9fbf0f601d7a4deb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fa6d37a0349742ad9fbf0f601d7a4deb 2024-11-07T12:25:05,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2c95e08f67bc47868663ad1dbb2fa711 2024-11-07T12:25:05,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/843b20c8267e4cd8a0591412e7ca712b 2024-11-07T12:25:05,303 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/166186f6ae484ab9804bc0fe919290ce to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/166186f6ae484ab9804bc0fe919290ce 2024-11-07T12:25:05,304 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/920a756841ce487aa90a8398dfb9f23e 2024-11-07T12:25:05,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/e53f1a5bbce448aa890f0d35a4f2584a 2024-11-07T12:25:05,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a78d6a515d8a428a9d25af8b3183de04 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/a78d6a515d8a428a9d25af8b3183de04 2024-11-07T12:25:05,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6fb66ba91f4744d5991f323bddc1aa0e 2024-11-07T12:25:05,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/b7a40678765c4807a02cfdca54019b68 2024-11-07T12:25:05,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/489de692e65c4cf3bcf0c9f7b467b609 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/489de692e65c4cf3bcf0c9f7b467b609 2024-11-07T12:25:05,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/4d39b62edaf74d2285bdc6fc6d9bbdab 2024-11-07T12:25:05,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/723c7db118ad4d138daa3171cebf915a 2024-11-07T12:25:05,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/fc3fabe5ecac4f5b8a241cb2ee5e5ac4 2024-11-07T12:25:05,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/9fe2688a9e844a48bb0f7366be0c4d18 2024-11-07T12:25:05,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/909bc58c7df34a6abd259ae8c43b3356 2024-11-07T12:25:05,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/8f3278100c664a0e835b7849a5cd9732 2024-11-07T12:25:05,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7e4e05fe988441f4b86ebae8162a4fe4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/c6a173fbfd714b9d9b21ce2dd3e7bcda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/bedfa061c8524631afce00343ff2949b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/de47400b7d5f4805af442711f315b98d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/51e94371475d45bdba4b94e1b22827dd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/4c4adc657e7749f0bc85f16a6913892d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b] to archive 2024-11-07T12:25:05,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:05,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/ebda12a300d9451bac2bb30fa9d34345 2024-11-07T12:25:05,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/704aeeee955042d7b15a39d9a4ce39b3 2024-11-07T12:25:05,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7e4e05fe988441f4b86ebae8162a4fe4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7e4e05fe988441f4b86ebae8162a4fe4 2024-11-07T12:25:05,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/88e95fa5e48847018292e4c02c8da642 2024-11-07T12:25:05,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/7971d8f48c044752b1d7512631b7ddb3 2024-11-07T12:25:05,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/f0d0bef0ae9148d58a6ad43ce611c810 2024-11-07T12:25:05,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/c6a173fbfd714b9d9b21ce2dd3e7bcda to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/c6a173fbfd714b9d9b21ce2dd3e7bcda 2024-11-07T12:25:05,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b23768a9246d49eaac7f0056b17911c2 2024-11-07T12:25:05,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6f91da6f9b6444bcad17721139edefbc 2024-11-07T12:25:05,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/bedfa061c8524631afce00343ff2949b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/bedfa061c8524631afce00343ff2949b 2024-11-07T12:25:05,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/0e3e4d73c7cf48e882bdd5775e622f7e 2024-11-07T12:25:05,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/9c150972314c4ebbb7cb9e72ee6ac948 2024-11-07T12:25:05,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/de47400b7d5f4805af442711f315b98d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/de47400b7d5f4805af442711f315b98d 2024-11-07T12:25:05,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3a4eba89d1d14938a79bba57780cc09c 2024-11-07T12:25:05,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f9b1f24ad30423c8a79fea77f721a15 2024-11-07T12:25:05,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/51e94371475d45bdba4b94e1b22827dd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/51e94371475d45bdba4b94e1b22827dd 2024-11-07T12:25:05,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3bc2513306404b04a75c85d0f3d51baf 2024-11-07T12:25:05,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a244b088b43f48bd8f2fa2fd0579f3a9 2024-11-07T12:25:05,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3f7a3c19cb4a47a4a94dc3a3ba4beedd 2024-11-07T12:25:05,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/60962122613e4dfcb25b7bd3b1e74af7 2024-11-07T12:25:05,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/afda486175b24a90a84612ace5cdeddd 2024-11-07T12:25:05,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/4c4adc657e7749f0bc85f16a6913892d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/4c4adc657e7749f0bc85f16a6913892d 2024-11-07T12:25:05,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/b7d1387fa48e45a6bae3464729126f51 2024-11-07T12:25:05,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/823c694d900746ceb93e4e5e19898756 2024-11-07T12:25:05,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8944cbb06cd2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/a593d6d949cf46f389cdeb4c1a30ff5b 2024-11-07T12:25:10,472 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-07T12:25:10,472 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:10,472 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,473 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:10,492 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-07T12:25:10,492 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:10,492 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,492 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 230 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1337 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4011 rows 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1339 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4010 rows 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1338 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4014 rows 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4019 rows 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1341 2024-11-07T12:25:10,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4016 rows 2024-11-07T12:25:10,492 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:25:10,492 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:25:10,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:10,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:10,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:25:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:10,496 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982310496"}]},"ts":"1730982310496"} 2024-11-07T12:25:10,497 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:25:10,497 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:25:10,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:25:10,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, UNASSIGN}] 2024-11-07T12:25:10,500 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, UNASSIGN 2024-11-07T12:25:10,500 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:10,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, UNASSIGN because future has completed 2024-11-07T12:25:10,502 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:25:10,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:10,654 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:10,654 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:25:10,654 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing e34bd2129ac88e7686bed4817db84e03, disabling compactions & flushes 2024-11-07T12:25:10,654 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:10,654 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:10,654 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. after waiting 0 ms 2024-11-07T12:25:10,654 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:10,655 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing e34bd2129ac88e7686bed4817db84e03 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T12:25:10,664 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735de4c75738c49f286690a379bccf218_e34bd2129ac88e7686bed4817db84e03 is 50, key is test_row_0/A:col10/1730982310471/Put/seqid=0 2024-11-07T12:25:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742274_1450 (size=9914) 2024-11-07T12:25:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:11,070 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:11,074 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735de4c75738c49f286690a379bccf218_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735de4c75738c49f286690a379bccf218_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:11,075 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/c904497b30c941bca14a18ccde58ed17, store: [table=TestAcidGuarantees family=A region=e34bd2129ac88e7686bed4817db84e03] 2024-11-07T12:25:11,076 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/c904497b30c941bca14a18ccde58ed17 is 175, key is test_row_0/A:col10/1730982310471/Put/seqid=0 2024-11-07T12:25:11,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742275_1451 (size=22561) 2024-11-07T12:25:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:11,485 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/c904497b30c941bca14a18ccde58ed17 2024-11-07T12:25:11,491 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/17ef1147a6ad4ed992cf0d56600e4904 is 50, key is test_row_0/B:col10/1730982310471/Put/seqid=0 2024-11-07T12:25:11,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742276_1452 (size=9857) 2024-11-07T12:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:11,895 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/17ef1147a6ad4ed992cf0d56600e4904 2024-11-07T12:25:11,902 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6e71bd44da4f4aa587d7ec41222e3ff2 is 50, key is test_row_0/C:col10/1730982310471/Put/seqid=0 2024-11-07T12:25:11,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742277_1453 (size=9857) 2024-11-07T12:25:12,306 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6e71bd44da4f4aa587d7ec41222e3ff2 2024-11-07T12:25:12,310 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/A/c904497b30c941bca14a18ccde58ed17 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/c904497b30c941bca14a18ccde58ed17 2024-11-07T12:25:12,313 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/c904497b30c941bca14a18ccde58ed17, entries=100, sequenceid=375, filesize=22.0 K 2024-11-07T12:25:12,313 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/B/17ef1147a6ad4ed992cf0d56600e4904 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/17ef1147a6ad4ed992cf0d56600e4904 2024-11-07T12:25:12,316 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/17ef1147a6ad4ed992cf0d56600e4904, entries=100, sequenceid=375, filesize=9.6 K 2024-11-07T12:25:12,316 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/.tmp/C/6e71bd44da4f4aa587d7ec41222e3ff2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6e71bd44da4f4aa587d7ec41222e3ff2 2024-11-07T12:25:12,319 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6e71bd44da4f4aa587d7ec41222e3ff2, entries=100, sequenceid=375, filesize=9.6 K 2024-11-07T12:25:12,320 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e34bd2129ac88e7686bed4817db84e03 in 1666ms, sequenceid=375, compaction requested=true 2024-11-07T12:25:12,324 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=4 2024-11-07T12:25:12,325 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. 2024-11-07T12:25:12,325 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for e34bd2129ac88e7686bed4817db84e03: Waiting for close lock at 1730982310654Running coprocessor pre-close hooks at 1730982310654Disabling compacts and flushes for region at 1730982310654Disabling writes for close at 1730982310654Obtaining lock to block concurrent updates at 1730982310655 (+1 ms)Preparing flush snapshotting stores in e34bd2129ac88e7686bed4817db84e03 at 1730982310655Finished memstore snapshotting TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730982310655Flushing stores of TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03. at 1730982310655Flushing e34bd2129ac88e7686bed4817db84e03/A: creating writer at 1730982310655Flushing mob file e34bd2129ac88e7686bed4817db84e03/A: appending metadata at 1730982310664 (+9 ms)Flushing mob file e34bd2129ac88e7686bed4817db84e03/A: closing flushed file at 1730982310664Flushing e34bd2129ac88e7686bed4817db84e03/A: appending metadata at 1730982311075 (+411 ms)Flushing e34bd2129ac88e7686bed4817db84e03/A: closing flushed file at 1730982311076 (+1 ms)Flushing e34bd2129ac88e7686bed4817db84e03/B: creating writer at 1730982311488 (+412 ms)Flushing e34bd2129ac88e7686bed4817db84e03/B: appending metadata at 1730982311491 (+3 ms)Flushing e34bd2129ac88e7686bed4817db84e03/B: closing flushed file at 1730982311491Flushing e34bd2129ac88e7686bed4817db84e03/C: creating writer at 1730982311899 (+408 ms)Flushing e34bd2129ac88e7686bed4817db84e03/C: appending metadata at 1730982311901 (+2 ms)Flushing e34bd2129ac88e7686bed4817db84e03/C: closing flushed file at 1730982311901Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195ccd75: reopening flushed file at 1730982312309 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428c8427: reopening flushed file at 1730982312313 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fce4676: reopening flushed file at 1730982312316 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e34bd2129ac88e7686bed4817db84e03 in 1666ms, sequenceid=375, compaction requested=true at 1730982312320 (+4 ms)Writing region close event to WAL at 1730982312322 (+2 ms)Running coprocessor post-close hooks at 1730982312325 (+3 ms)Closed at 1730982312325 2024-11-07T12:25:12,327 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,327 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e34bd2129ac88e7686bed4817db84e03, regionState=CLOSED 2024-11-07T12:25:12,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:12,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T12:25:12,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure e34bd2129ac88e7686bed4817db84e03, server=8944cbb06cd2,46221,1730982182218 in 1.8270 sec 2024-11-07T12:25:12,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-07T12:25:12,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e34bd2129ac88e7686bed4817db84e03, UNASSIGN in 1.8320 sec 2024-11-07T12:25:12,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T12:25:12,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8340 sec 2024-11-07T12:25:12,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982312334"}]},"ts":"1730982312334"} 2024-11-07T12:25:12,335 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:25:12,335 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:25:12,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8430 sec 2024-11-07T12:25:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T12:25:12,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:12,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:25:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,644 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:25:12,645 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,651 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits] 2024-11-07T12:25:12,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/03a9625c5a56479fa69c72167a776698 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/03a9625c5a56479fa69c72167a776698 2024-11-07T12:25:12,655 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5102f6169ee2462782135e7392599bde to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/5102f6169ee2462782135e7392599bde 2024-11-07T12:25:12,656 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/c904497b30c941bca14a18ccde58ed17 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/A/c904497b30c941bca14a18ccde58ed17 2024-11-07T12:25:12,658 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/17ef1147a6ad4ed992cf0d56600e4904 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/17ef1147a6ad4ed992cf0d56600e4904 2024-11-07T12:25:12,659 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e1831246db94f8e8ea716185992cef4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/2e1831246db94f8e8ea716185992cef4 2024-11-07T12:25:12,660 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c4d013cf42947dfa34ac9f30d49f9e7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/B/6c4d013cf42947dfa34ac9f30d49f9e7 2024-11-07T12:25:12,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3e1204f6420d4105b3e9d9ba9f8b1d27 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/3e1204f6420d4105b3e9d9ba9f8b1d27 2024-11-07T12:25:12,663 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/40839fe2cf7e42159cfd741de38622c5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/40839fe2cf7e42159cfd741de38622c5 2024-11-07T12:25:12,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6e71bd44da4f4aa587d7ec41222e3ff2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/C/6e71bd44da4f4aa587d7ec41222e3ff2 2024-11-07T12:25:12,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits/378.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03/recovered.edits/378.seqid 2024-11-07T12:25:12,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,666 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:25:12,667 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:25:12,667 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:25:12,671 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110706f87fef55214c2f988783a7d9c87006_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110706f87fef55214c2f988783a7d9c87006_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,672 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070cb9746d441f4beabf8e1e660027834d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070cb9746d441f4beabf8e1e660027834d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,673 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712d2f96988da48debbafabfbf77c1a6d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712d2f96988da48debbafabfbf77c1a6d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,674 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071f152f1c707e432aa10989ca8989bb70_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071f152f1c707e432aa10989ca8989bb70_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,675 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735de4c75738c49f286690a379bccf218_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735de4c75738c49f286690a379bccf218_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,676 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735f2799c7a4c4c15a83df2e3df551115_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110735f2799c7a4c4c15a83df2e3df551115_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,677 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738350da9f4c44aaba5193713f38912b5_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738350da9f4c44aaba5193713f38912b5_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,678 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074c602ac41de24f908d26b0336a1c32fb_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074c602ac41de24f908d26b0336a1c32fb_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,679 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075679f9e7d8304d62b38f80258a956d71_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075679f9e7d8304d62b38f80258a956d71_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,680 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bc4fa804b72410c8f7b1a8822a3c43f_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bc4fa804b72410c8f7b1a8822a3c43f_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107753ca431e9d44b3ab3f8f2bb367a9f0d_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107753ca431e9d44b3ab3f8f2bb367a9f0d_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,681 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110779bc874bb5f14e2394ce8431df314011_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110779bc874bb5f14e2394ce8431df314011_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,682 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786364601e2784b4599fcd5b90d3c1fa0_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786364601e2784b4599fcd5b90d3c1fa0_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,683 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c321c056d4943b8ab98bfb1202c8af7_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c321c056d4943b8ab98bfb1202c8af7_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,684 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d14acca2d4846c59c34b05d721d366f_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d14acca2d4846c59c34b05d721d366f_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,685 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079042f988bbad40c697f0e4b53d2393eb_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079042f988bbad40c697f0e4b53d2393eb_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,687 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a6d6a6b6c504ff5b0282e77e28cb6b2_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a6d6a6b6c504ff5b0282e77e28cb6b2_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,688 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c98cf25949a246d99eaf5f6f23d523a2_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c98cf25949a246d99eaf5f6f23d523a2_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,689 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc2ee6c593f749e88cba645ebdab21e7_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cc2ee6c593f749e88cba645ebdab21e7_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfbcc86bd0ff40bd8aca871c5a3d9196_e34bd2129ac88e7686bed4817db84e03 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfbcc86bd0ff40bd8aca871c5a3d9196_e34bd2129ac88e7686bed4817db84e03 2024-11-07T12:25:12,690 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:25:12,693 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,695 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:25:12,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:25:12,698 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,698 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:25:12,699 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982312698"}]},"ts":"9223372036854775807"} 2024-11-07T12:25:12,701 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:25:12,701 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e34bd2129ac88e7686bed4817db84e03, NAME => 'TestAcidGuarantees,,1730982276584.e34bd2129ac88e7686bed4817db84e03.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:25:12,701 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:25:12,701 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982312701"}]},"ts":"9223372036854775807"} 2024-11-07T12:25:12,702 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:25:12,703 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-07T12:25:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T12:25:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:25:12,754 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:12,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 242), OpenFileDescriptor=445 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=561 (was 600), ProcessCount=11 (was 11), AvailableMemoryMB=1603 (was 1491) - AvailableMemoryMB LEAK? - 2024-11-07T12:25:12,769 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=561, ProcessCount=11, AvailableMemoryMB=1602 2024-11-07T12:25:12,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:25:12,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:25:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:12,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:25:12,773 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:12,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-07T12:25:12,774 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:25:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742278_1454 (size=959) 2024-11-07T12:25:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:13,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:25:13,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742279_1455 (size=53) 2024-11-07T12:25:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:13,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:25:13,586 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ea4184a46d62bfe1df974384dfe56d56, disabling compactions & flushes 2024-11-07T12:25:13,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. after waiting 0 ms 2024-11-07T12:25:13,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,587 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ea4184a46d62bfe1df974384dfe56d56: Waiting for close lock at 1730982313586Disabling compacts and flushes for region at 1730982313586Disabling writes for close at 1730982313587 (+1 ms)Writing region close event to WAL at 1730982313587Closed at 1730982313587 2024-11-07T12:25:13,588 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:25:13,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982313588"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982313588"}]},"ts":"1730982313588"} 2024-11-07T12:25:13,590 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:25:13,591 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:25:13,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982313591"}]},"ts":"1730982313591"} 2024-11-07T12:25:13,593 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:25:13,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, ASSIGN}] 2024-11-07T12:25:13,595 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, ASSIGN 2024-11-07T12:25:13,596 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:25:13,746 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ea4184a46d62bfe1df974384dfe56d56, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:13,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, ASSIGN because future has completed 2024-11-07T12:25:13,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:13,905 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,905 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:25:13,906 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,906 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:25:13,906 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,906 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,907 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,909 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4184a46d62bfe1df974384dfe56d56 columnFamilyName A 2024-11-07T12:25:13,909 DEBUG [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:13,909 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(327): Store=ea4184a46d62bfe1df974384dfe56d56/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:13,909 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,911 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4184a46d62bfe1df974384dfe56d56 columnFamilyName B 2024-11-07T12:25:13,911 DEBUG [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:13,911 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(327): Store=ea4184a46d62bfe1df974384dfe56d56/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:13,911 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,913 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4184a46d62bfe1df974384dfe56d56 columnFamilyName C 2024-11-07T12:25:13,913 DEBUG [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:13,913 INFO [StoreOpener-ea4184a46d62bfe1df974384dfe56d56-1 {}] regionserver.HStore(327): Store=ea4184a46d62bfe1df974384dfe56d56/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:13,913 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,914 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,914 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,915 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,915 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,915 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:25:13,916 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,918 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:25:13,919 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened ea4184a46d62bfe1df974384dfe56d56; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73854928, jitterRate=0.1005241870880127}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:25:13,919 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:13,920 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for ea4184a46d62bfe1df974384dfe56d56: Running coprocessor pre-open hook at 1730982313906Writing region info on filesystem at 1730982313906Initializing all the Stores at 1730982313907 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982313907Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982313907Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982313907Cleaning up temporary data from old regions at 1730982313915 (+8 ms)Running coprocessor post-open hooks at 1730982313919 (+4 ms)Region opened successfully at 1730982313920 (+1 ms) 2024-11-07T12:25:13,921 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., pid=118, masterSystemTime=1730982313900 2024-11-07T12:25:13,922 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,922 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:13,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ea4184a46d62bfe1df974384dfe56d56, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:13,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:13,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-07T12:25:13,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 in 177 msec 2024-11-07T12:25:13,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-07T12:25:13,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, ASSIGN in 333 msec 2024-11-07T12:25:13,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:25:13,928 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982313928"}]},"ts":"1730982313928"} 2024-11-07T12:25:13,929 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:25:13,930 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:25:13,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-07T12:25:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-07T12:25:14,922 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:14,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16af2234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,925 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e5a24b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3159e353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,927 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:25:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:25:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35a981b2 2024-11-07T12:25:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:25:14,931 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:25:14,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695dbdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,932 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716f3ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646b121c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594674bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d26b88d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ebff40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,937 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf2768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,938 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9aab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,940 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259fd7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,941 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649faddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640ad474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22df2464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ae0787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e3ff17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37de5ffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c246de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,950 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2742ac3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33527d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2cf336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45da326c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,957 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f15bfdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b33a176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:14,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:14,959 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17589d46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:14,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1124f29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:14,962 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,962 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,962 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,963 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,963 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,963 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,964 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,966 DEBUG [Thread-2040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,966 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:14,967 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,967 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:14,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-07T12:25:14,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:14,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2] 2024-11-07T12:25:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:14,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:25:15,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/c8b78ea1718d4dd2972879d3d5ccef59 is 50, key is test_row_0/A:col10/1730982314977/Put/seqid=0 2024-11-07T12:25:15,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742280_1456 (size=12001) 2024-11-07T12:25:15,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982375010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982375040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982375040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982375040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982375040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:15,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982375151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982375151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982375151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982375152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982375161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982375365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982375365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982375365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982375366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982375371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/c8b78ea1718d4dd2972879d3d5ccef59 2024-11-07T12:25:15,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/2d3bb1080b1c4b589f33aaf45c02310b is 50, key is test_row_0/B:col10/1730982314977/Put/seqid=0 2024-11-07T12:25:15,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742281_1457 (size=12001) 2024-11-07T12:25:15,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/2d3bb1080b1c4b589f33aaf45c02310b 2024-11-07T12:25:15,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/7221ab34bf4b4e9488682c013a4caa73 is 50, key is test_row_0/C:col10/1730982314977/Put/seqid=0 2024-11-07T12:25:15,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742282_1458 (size=12001) 2024-11-07T12:25:15,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982375680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982375681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982375681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982375681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982375681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:15,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/7221ab34bf4b4e9488682c013a4caa73 2024-11-07T12:25:15,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/c8b78ea1718d4dd2972879d3d5ccef59 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59 2024-11-07T12:25:15,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:15,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:15,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:25:15,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/2d3bb1080b1c4b589f33aaf45c02310b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b 2024-11-07T12:25:15,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:25:15,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/7221ab34bf4b4e9488682c013a4caa73 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73 2024-11-07T12:25:15,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T12:25:15,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ea4184a46d62bfe1df974384dfe56d56 in 932ms, sequenceid=13, compaction requested=false 2024-11-07T12:25:15,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:16,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T12:25:16,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:16,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:25:16,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/59c98416736c4414871e9baf2f77a0f4 is 50, key is test_row_0/A:col10/1730982315016/Put/seqid=0 2024-11-07T12:25:16,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742283_1459 (size=12001) 2024-11-07T12:25:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:16,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982376208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982376208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982376208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982376209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982376209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982376320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982376321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982376321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982376321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982376321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,461 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/59c98416736c4414871e9baf2f77a0f4 2024-11-07T12:25:16,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/577bf2111ba04a12a5f8483bcf56fc25 is 50, key is test_row_0/B:col10/1730982315016/Put/seqid=0 2024-11-07T12:25:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742284_1460 (size=12001) 2024-11-07T12:25:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982376530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982376530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982376531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982376531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982376531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982376841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982376841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982376841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982376841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982376842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:16,882 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/577bf2111ba04a12a5f8483bcf56fc25 2024-11-07T12:25:16,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/cc865f0e1f5c488a8d7ba45854d467c3 is 50, key is test_row_0/C:col10/1730982315016/Put/seqid=0 2024-11-07T12:25:16,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742285_1461 (size=12001) 2024-11-07T12:25:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:17,291 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/cc865f0e1f5c488a8d7ba45854d467c3 2024-11-07T12:25:17,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/59c98416736c4414871e9baf2f77a0f4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4 2024-11-07T12:25:17,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:25:17,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/577bf2111ba04a12a5f8483bcf56fc25 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25 2024-11-07T12:25:17,313 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:25:17,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/cc865f0e1f5c488a8d7ba45854d467c3 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3 2024-11-07T12:25:17,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T12:25:17,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ea4184a46d62bfe1df974384dfe56d56 in 1269ms, sequenceid=37, compaction requested=false 2024-11-07T12:25:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:17,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-07T12:25:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-07T12:25:17,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T12:25:17,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-11-07T12:25:17,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3560 sec 2024-11-07T12:25:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:17,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:17,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T12:25:17,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/41a9801dc7dc42878b090225971bf892 is 50, key is test_row_0/A:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742286_1462 (size=14341) 2024-11-07T12:25:17,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/41a9801dc7dc42878b090225971bf892 2024-11-07T12:25:17,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 is 50, key is test_row_0/B:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742287_1463 (size=12001) 2024-11-07T12:25:17,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 2024-11-07T12:25:17,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/82779f30a1d04237a851cbe768386949 is 50, key is test_row_0/C:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742288_1464 (size=12001) 2024-11-07T12:25:17,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/82779f30a1d04237a851cbe768386949 2024-11-07T12:25:17,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982377405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982377405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982377412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/41a9801dc7dc42878b090225971bf892 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892 2024-11-07T12:25:17,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982377417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892, entries=200, sequenceid=50, filesize=14.0 K 2024-11-07T12:25:17,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 2024-11-07T12:25:17,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982377418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:25:17,430 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T12:25:17,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/82779f30a1d04237a851cbe768386949 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949 2024-11-07T12:25:17,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T12:25:17,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ea4184a46d62bfe1df974384dfe56d56 in 77ms, sequenceid=50, compaction requested=true 2024-11-07T12:25:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:17,438 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:17,438 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:17,438 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:17,438 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=37.4 K 2024-11-07T12:25:17,438 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.2 K 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b78ea1718d4dd2972879d3d5ccef59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730982314975 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3bb1080b1c4b589f33aaf45c02310b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730982314975 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c98416736c4414871e9baf2f77a0f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982315016 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 577bf2111ba04a12a5f8483bcf56fc25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982315016 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a9801dc7dc42878b090225971bf892, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:17,439 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f5eaa756a1cf4a03ae5dce9e1631c2d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:17,448 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:17,448 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:17,449 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/e33cc6505d024484ad4d6a0acccd3ad2 is 50, key is test_row_0/A:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,449 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/bf6b08da7245435aa22e146250baa11c is 50, key is test_row_0/B:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742289_1465 (size=12104) 2024-11-07T12:25:17,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742290_1466 (size=12104) 2024-11-07T12:25:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:17,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:17,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:25:17,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/d350ce4d8e8649919682054b866c6f54 is 50, key is test_row_0/A:col10/1730982317416/Put/seqid=0 2024-11-07T12:25:17,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982377544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742291_1467 (size=12001) 2024-11-07T12:25:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982377555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982377558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982377661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982377670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982377670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982377681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982377681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,854 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:25:17,860 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/bf6b08da7245435aa22e146250baa11c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/bf6b08da7245435aa22e146250baa11c 2024-11-07T12:25:17,860 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/e33cc6505d024484ad4d6a0acccd3ad2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/e33cc6505d024484ad4d6a0acccd3ad2 2024-11-07T12:25:17,866 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into bf6b08da7245435aa22e146250baa11c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:17,866 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:17,866 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982317438; duration=0sec 2024-11-07T12:25:17,866 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:17,866 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:17,866 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:17,867 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:17,868 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:17,868 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:17,868 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.2 K 2024-11-07T12:25:17,869 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 7221ab34bf4b4e9488682c013a4caa73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730982314975 2024-11-07T12:25:17,869 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting cc865f0e1f5c488a8d7ba45854d467c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730982315016 2024-11-07T12:25:17,870 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 82779f30a1d04237a851cbe768386949, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:17,876 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into e33cc6505d024484ad4d6a0acccd3ad2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:17,877 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:17,877 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982317437; duration=0sec 2024-11-07T12:25:17,877 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:17,877 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:17,879 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:17,880 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2365c66b71ff4e8c8f1d58274d0b3463 is 50, key is test_row_0/C:col10/1730982316207/Put/seqid=0 2024-11-07T12:25:17,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982377870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982377881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982377881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742292_1468 (size=12104) 2024-11-07T12:25:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982377891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982377891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:17,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/d350ce4d8e8649919682054b866c6f54 2024-11-07T12:25:17,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/887b1e46fb474649bcd4b03c3fef66ba is 50, key is test_row_0/B:col10/1730982317416/Put/seqid=0 2024-11-07T12:25:17,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742293_1469 (size=12001) 2024-11-07T12:25:17,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/887b1e46fb474649bcd4b03c3fef66ba 2024-11-07T12:25:17,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/a9beee83beff4fb78ca21bea145ced42 is 50, key is test_row_0/C:col10/1730982317416/Put/seqid=0 2024-11-07T12:25:17,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742294_1470 (size=12001) 2024-11-07T12:25:18,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982378190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982378190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982378191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982378201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982378201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,290 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2365c66b71ff4e8c8f1d58274d0b3463 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2365c66b71ff4e8c8f1d58274d0b3463 2024-11-07T12:25:18,293 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into 2365c66b71ff4e8c8f1d58274d0b3463(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:18,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:18,293 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982317438; duration=0sec 2024-11-07T12:25:18,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:18,293 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:18,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/a9beee83beff4fb78ca21bea145ced42 2024-11-07T12:25:18,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/d350ce4d8e8649919682054b866c6f54 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54 2024-11-07T12:25:18,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:25:18,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/887b1e46fb474649bcd4b03c3fef66ba as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba 2024-11-07T12:25:18,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:25:18,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/a9beee83beff4fb78ca21bea145ced42 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42 2024-11-07T12:25:18,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T12:25:18,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ea4184a46d62bfe1df974384dfe56d56 in 856ms, sequenceid=75, compaction requested=false 2024-11-07T12:25:18,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:18,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:18,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:25:18,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/05c886c0c49c4ad0bb0ab45510ef1312 is 50, key is test_row_0/A:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:18,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742295_1471 (size=16681) 2024-11-07T12:25:18,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982378743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982378744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982378749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982378750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982378861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982378861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982378861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982378861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982378871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982379081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982379081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982379081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982379081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982379090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/05c886c0c49c4ad0bb0ab45510ef1312 2024-11-07T12:25:19,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/4987438bd30f4b4c8bd1ace6746bb572 is 50, key is test_row_0/B:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T12:25:19,131 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:19,131 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-07T12:25:19,133 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:19,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742296_1472 (size=12001) 2024-11-07T12:25:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:19,134 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:19,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:19,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T12:25:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982379391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982379391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982379391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982379401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982379411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T12:25:19,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:19,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:19,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/4987438bd30f4b4c8bd1ace6746bb572 2024-11-07T12:25:19,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/647958876659463fbc54abb2af06edcf is 50, key is test_row_0/C:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:19,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742297_1473 (size=12001) 2024-11-07T12:25:19,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/647958876659463fbc54abb2af06edcf 2024-11-07T12:25:19,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/05c886c0c49c4ad0bb0ab45510ef1312 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312 2024-11-07T12:25:19,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312, entries=250, sequenceid=91, filesize=16.3 K 2024-11-07T12:25:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/4987438bd30f4b4c8bd1ace6746bb572 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572 2024-11-07T12:25:19,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:25:19,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/647958876659463fbc54abb2af06edcf as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf 2024-11-07T12:25:19,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T12:25:19,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ea4184a46d62bfe1df974384dfe56d56 in 873ms, sequenceid=91, compaction requested=true 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:19,587 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:19,587 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:19,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:19,588 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:19,588 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:19,588 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,588 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/bf6b08da7245435aa22e146250baa11c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.3 K 2024-11-07T12:25:19,589 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:19,589 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:19,589 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,590 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/e33cc6505d024484ad4d6a0acccd3ad2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=39.8 K 2024-11-07T12:25:19,590 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6b08da7245435aa22e146250baa11c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:19,590 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33cc6505d024484ad4d6a0acccd3ad2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:19,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 887b1e46fb474649bcd4b03c3fef66ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982317416 2024-11-07T12:25:19,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d350ce4d8e8649919682054b866c6f54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982317416 2024-11-07T12:25:19,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4987438bd30f4b4c8bd1ace6746bb572, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:19,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c886c0c49c4ad0bb0ab45510ef1312, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:19,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T12:25:19,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,596 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:25:19,596 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:19,597 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/000a67f55ca14c4292ef8ab538d3e29c is 50, key is test_row_0/A:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:19,600 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:19,600 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/cd9d7f6822e147cfb3b9914efb7220d5 is 50, key is test_row_0/B:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/b64d2a56a944473987a40642f625b48b is 50, key is test_row_0/A:col10/1730982318734/Put/seqid=0 2024-11-07T12:25:19,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742298_1474 (size=12207) 2024-11-07T12:25:19,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742299_1475 (size=12207) 2024-11-07T12:25:19,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742300_1476 (size=12001) 2024-11-07T12:25:19,644 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/cd9d7f6822e147cfb3b9914efb7220d5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/cd9d7f6822e147cfb3b9914efb7220d5 2024-11-07T12:25:19,650 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into cd9d7f6822e147cfb3b9914efb7220d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:19,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:19,650 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982319587; duration=0sec 2024-11-07T12:25:19,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:19,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:19,650 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:19,651 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:19,651 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:19,651 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:19,651 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2365c66b71ff4e8c8f1d58274d0b3463, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.3 K 2024-11-07T12:25:19,651 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2365c66b71ff4e8c8f1d58274d0b3463, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730982316206 2024-11-07T12:25:19,652 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a9beee83beff4fb78ca21bea145ced42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730982317416 2024-11-07T12:25:19,652 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 647958876659463fbc54abb2af06edcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:19,666 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:19,667 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/fa5d49bdd1954e26a6240499156f0614 is 50, key is test_row_0/C:col10/1730982318713/Put/seqid=0 2024-11-07T12:25:19,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742301_1477 (size=12207) 2024-11-07T12:25:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:19,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982379932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982379932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982379932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982379939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982379940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,021 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/000a67f55ca14c4292ef8ab538d3e29c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/000a67f55ca14c4292ef8ab538d3e29c 2024-11-07T12:25:20,024 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 000a67f55ca14c4292ef8ab538d3e29c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:20,024 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:20,024 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982319587; duration=0sec 2024-11-07T12:25:20,024 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:20,024 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:20,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/b64d2a56a944473987a40642f625b48b 2024-11-07T12:25:20,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/6106970c9bea49d09fac1aae4c48faba is 50, key is test_row_0/B:col10/1730982318734/Put/seqid=0 2024-11-07T12:25:20,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742302_1478 (size=12001) 2024-11-07T12:25:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982380051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982380051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982380051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982380051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,083 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/fa5d49bdd1954e26a6240499156f0614 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/fa5d49bdd1954e26a6240499156f0614 2024-11-07T12:25:20,088 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into fa5d49bdd1954e26a6240499156f0614(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:20,088 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:20,088 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982319587; duration=0sec 2024-11-07T12:25:20,088 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:20,088 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:20,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982380261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982380261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982380261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982380261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:20,446 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/6106970c9bea49d09fac1aae4c48faba 2024-11-07T12:25:20,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/9d26703e61f148b889f2500d99e18f24 is 50, key is test_row_0/C:col10/1730982318734/Put/seqid=0 2024-11-07T12:25:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742303_1479 (size=12001) 2024-11-07T12:25:20,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982380571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982380571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982380580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982380580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:20,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/9d26703e61f148b889f2500d99e18f24 2024-11-07T12:25:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/b64d2a56a944473987a40642f625b48b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b 2024-11-07T12:25:20,869 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T12:25:20,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/6106970c9bea49d09fac1aae4c48faba as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba 2024-11-07T12:25:20,876 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T12:25:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/9d26703e61f148b889f2500d99e18f24 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24 2024-11-07T12:25:20,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T12:25:20,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ea4184a46d62bfe1df974384dfe56d56 in 1294ms, sequenceid=114, compaction requested=false 2024-11-07T12:25:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-07T12:25:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-07T12:25:20,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T12:25:20,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7590 sec 2024-11-07T12:25:20,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.7650 sec 2024-11-07T12:25:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:20,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:20,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:25:20,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f891816cc13a47b9867f2b62eee6023f is 50, key is test_row_0/A:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:20,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742304_1480 (size=12047) 2024-11-07T12:25:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982381090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982381090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982381090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982381102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982381103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982381221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982381221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T12:25:21,281 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:21,281 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-07T12:25:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:21,282 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:21,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:21,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:21,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f891816cc13a47b9867f2b62eee6023f 2024-11-07T12:25:21,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/45cbf8203cd046bfbc2453f210687fe1 is 50, key is test_row_0/B:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742305_1481 (size=9707) 2024-11-07T12:25:21,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/45cbf8203cd046bfbc2453f210687fe1 2024-11-07T12:25:21,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/385576103b224d33848f0ebe04cce4ce is 50, key is test_row_0/C:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:21,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742306_1482 (size=9707) 2024-11-07T12:25:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:21,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T12:25:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982381430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982381431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T12:25:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:21,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T12:25:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982381741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982381750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:21,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/385576103b224d33848f0ebe04cce4ce 2024-11-07T12:25:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f891816cc13a47b9867f2b62eee6023f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f 2024-11-07T12:25:21,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T12:25:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/45cbf8203cd046bfbc2453f210687fe1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1 2024-11-07T12:25:21,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1, entries=100, sequenceid=131, filesize=9.5 K 2024-11-07T12:25:21,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/385576103b224d33848f0ebe04cce4ce as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce 2024-11-07T12:25:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce, entries=100, sequenceid=131, filesize=9.5 K 2024-11-07T12:25:21,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ea4184a46d62bfe1df974384dfe56d56 in 844ms, sequenceid=131, compaction requested=true 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:21,801 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:21,801 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:21,801 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:21,801 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:21,802 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,802 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/cd9d7f6822e147cfb3b9914efb7220d5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=33.1 K 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:21,802 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,802 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/000a67f55ca14c4292ef8ab538d3e29c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.4 K 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9d7f6822e147cfb3b9914efb7220d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000a67f55ca14c4292ef8ab538d3e29c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6106970c9bea49d09fac1aae4c48faba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982318734 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 45cbf8203cd046bfbc2453f210687fe1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982319937 2024-11-07T12:25:21,802 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b64d2a56a944473987a40642f625b48b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982318734 2024-11-07T12:25:21,803 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f891816cc13a47b9867f2b62eee6023f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982319937 2024-11-07T12:25:21,816 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:21,817 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/eee1a8a9eaa4479ead834ab65bde2c3f is 50, key is test_row_0/B:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:21,817 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:21,817 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/382b4e25ef00412c85012e9654364353 is 50, key is test_row_0/A:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:21,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742307_1483 (size=12359) 2024-11-07T12:25:21,828 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/382b4e25ef00412c85012e9654364353 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/382b4e25ef00412c85012e9654364353 2024-11-07T12:25:21,835 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 382b4e25ef00412c85012e9654364353(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:21,835 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:21,835 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982321801; duration=0sec 2024-11-07T12:25:21,835 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:21,835 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:21,835 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:21,836 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:21,836 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:21,836 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,836 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/fa5d49bdd1954e26a6240499156f0614, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=33.1 K 2024-11-07T12:25:21,837 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5d49bdd1954e26a6240499156f0614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730982317541 2024-11-07T12:25:21,837 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d26703e61f148b889f2500d99e18f24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730982318734 2024-11-07T12:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742308_1484 (size=12359) 2024-11-07T12:25:21,838 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385576103b224d33848f0ebe04cce4ce, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982319937 2024-11-07T12:25:21,842 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/eee1a8a9eaa4479ead834ab65bde2c3f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/eee1a8a9eaa4479ead834ab65bde2c3f 2024-11-07T12:25:21,847 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into eee1a8a9eaa4479ead834ab65bde2c3f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:21,847 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:21,847 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982321801; duration=0sec 2024-11-07T12:25:21,847 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:21,847 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:21,849 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:21,849 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/622556e932c54d32841145b585b5d63b is 50, key is test_row_0/C:col10/1730982320955/Put/seqid=0 2024-11-07T12:25:21,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742309_1485 (size=12359) 2024-11-07T12:25:21,865 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/622556e932c54d32841145b585b5d63b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/622556e932c54d32841145b585b5d63b 2024-11-07T12:25:21,873 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into 622556e932c54d32841145b585b5d63b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:21,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:21,873 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982321801; duration=0sec 2024-11-07T12:25:21,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:21,873 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:21,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T12:25:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:21,893 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:25:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/5c1c827f62ef44aa8604bb7503bc3306 is 50, key is test_row_0/A:col10/1730982321083/Put/seqid=0 2024-11-07T12:25:21,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742310_1486 (size=12151) 2024-11-07T12:25:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:22,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:22,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982382133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982382134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982382139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982382251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982382251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982382252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982382260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982382271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/5c1c827f62ef44aa8604bb7503bc3306 2024-11-07T12:25:22,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e88b4e31a61e42689be34285f8d4b0b9 is 50, key is test_row_0/B:col10/1730982321083/Put/seqid=0 2024-11-07T12:25:22,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742311_1487 (size=12151) 2024-11-07T12:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982382460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982382460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982382460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,713 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e88b4e31a61e42689be34285f8d4b0b9 2024-11-07T12:25:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ae2d7991cd01443b91db25a5ff3828ca is 50, key is test_row_0/C:col10/1730982321083/Put/seqid=0 2024-11-07T12:25:22,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742312_1488 (size=12151) 2024-11-07T12:25:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982382771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982382771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982382781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,124 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ae2d7991cd01443b91db25a5ff3828ca 2024-11-07T12:25:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/5c1c827f62ef44aa8604bb7503bc3306 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306 2024-11-07T12:25:23,130 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:25:23,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e88b4e31a61e42689be34285f8d4b0b9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9 2024-11-07T12:25:23,135 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:25:23,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ae2d7991cd01443b91db25a5ff3828ca as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca 2024-11-07T12:25:23,140 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T12:25:23,141 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ea4184a46d62bfe1df974384dfe56d56 in 1248ms, sequenceid=156, compaction requested=false 2024-11-07T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-07T12:25:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-07T12:25:23,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-07T12:25:23,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-07T12:25:23,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8630 sec 2024-11-07T12:25:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:23,277 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:23,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:25:23,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/6f7f4d1936c44c8aa9818f0752bba767 is 50, key is test_row_0/A:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742313_1489 (size=14541) 2024-11-07T12:25:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982383314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982383315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982383321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982383324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982383325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T12:25:23,430 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:23,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-07T12:25:23,432 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:23,433 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:23,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982383430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982383430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982383431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982383441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982383441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:23,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T12:25:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982383641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982383641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982383641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982383650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982383651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/6f7f4d1936c44c8aa9818f0752bba767 2024-11-07T12:25:23,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/c28c45785a474fd48fcbcdd4bc3221f4 is 50, key is test_row_0/B:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:23,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742314_1490 (size=12151) 2024-11-07T12:25:23,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/c28c45785a474fd48fcbcdd4bc3221f4 2024-11-07T12:25:23,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d0dc1b188c874572a523811ee5a046ec is 50, key is test_row_0/C:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:23,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T12:25:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742315_1491 (size=12151) 2024-11-07T12:25:23,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T12:25:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:23,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:23,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982383950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982383950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982383951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982383961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982383961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T12:25:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:24,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:24,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d0dc1b188c874572a523811ee5a046ec 2024-11-07T12:25:24,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/6f7f4d1936c44c8aa9818f0752bba767 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767 2024-11-07T12:25:24,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767, entries=200, sequenceid=171, filesize=14.2 K 2024-11-07T12:25:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/c28c45785a474fd48fcbcdd4bc3221f4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4 2024-11-07T12:25:24,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T12:25:24,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d0dc1b188c874572a523811ee5a046ec as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec 2024-11-07T12:25:24,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T12:25:24,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ea4184a46d62bfe1df974384dfe56d56 in 896ms, sequenceid=171, compaction requested=true 2024-11-07T12:25:24,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:24,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:24,174 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:24,174 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,174 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,174 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/382b4e25ef00412c85012e9654364353, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=38.1 K 2024-11-07T12:25:24,174 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/eee1a8a9eaa4479ead834ab65bde2c3f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.8 K 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 382b4e25ef00412c85012e9654364353, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982318749 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting eee1a8a9eaa4479ead834ab65bde2c3f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982318749 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1c827f62ef44aa8604bb7503bc3306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982321083 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e88b4e31a61e42689be34285f8d4b0b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982321083 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f7f4d1936c44c8aa9818f0752bba767, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:24,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting c28c45785a474fd48fcbcdd4bc3221f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:24,181 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:24,181 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:24,181 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/ef7244d04edd4e05876de9c29e201a1b is 50, key is test_row_0/B:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:24,182 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/c8af16b19de34576862ed73dc2a92e29 is 50, key is test_row_0/A:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742317_1493 (size=12561) 2024-11-07T12:25:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742316_1492 (size=12561) 2024-11-07T12:25:24,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T12:25:24,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,199 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:25:24,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f8c45c3822e74db29dc00e6f5d894086 is 50, key is test_row_0/A:col10/1730982323324/Put/seqid=0 2024-11-07T12:25:24,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742318_1494 (size=12151) 2024-11-07T12:25:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:24,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:24,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982384486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982384492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982384492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982384493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982384493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:24,588 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/ef7244d04edd4e05876de9c29e201a1b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/ef7244d04edd4e05876de9c29e201a1b 2024-11-07T12:25:24,588 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/c8af16b19de34576862ed73dc2a92e29 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8af16b19de34576862ed73dc2a92e29 2024-11-07T12:25:24,591 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into ef7244d04edd4e05876de9c29e201a1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:24,591 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into c8af16b19de34576862ed73dc2a92e29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:24,591 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982324174; duration=0sec 2024-11-07T12:25:24,591 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982324173; duration=0sec 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:24,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:24,592 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:24,592 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:24,592 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:24,592 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/622556e932c54d32841145b585b5d63b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=35.8 K 2024-11-07T12:25:24,592 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 622556e932c54d32841145b585b5d63b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730982318749 2024-11-07T12:25:24,592 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2d7991cd01443b91db25a5ff3828ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730982321083 2024-11-07T12:25:24,592 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d0dc1b188c874572a523811ee5a046ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:24,598 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:24,598 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/09a14c9cd6114398875d66d4dc9d1240 is 50, key is test_row_0/C:col10/1730982322133/Put/seqid=0 2024-11-07T12:25:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742319_1495 (size=12561) 2024-11-07T12:25:24,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f8c45c3822e74db29dc00e6f5d894086 2024-11-07T12:25:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982384601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f72e20359a87488aaa3611bd5c7f3da5 is 50, key is test_row_0/B:col10/1730982323324/Put/seqid=0 2024-11-07T12:25:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982384611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982384611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982384611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982384611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742320_1496 (size=12151) 2024-11-07T12:25:24,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982384811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982384821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982384821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982384821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982384830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,004 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/09a14c9cd6114398875d66d4dc9d1240 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/09a14c9cd6114398875d66d4dc9d1240 2024-11-07T12:25:25,013 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into 09a14c9cd6114398875d66d4dc9d1240(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:25,013 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:25,013 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982324174; duration=0sec 2024-11-07T12:25:25,014 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:25,014 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:25,021 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f72e20359a87488aaa3611bd5c7f3da5 2024-11-07T12:25:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2c29738f993c443c92543d4f4d79e753 is 50, key is test_row_0/C:col10/1730982323324/Put/seqid=0 2024-11-07T12:25:25,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742321_1497 (size=12151) 2024-11-07T12:25:25,036 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2c29738f993c443c92543d4f4d79e753 2024-11-07T12:25:25,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/f8c45c3822e74db29dc00e6f5d894086 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086 2024-11-07T12:25:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T12:25:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f72e20359a87488aaa3611bd5c7f3da5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5 2024-11-07T12:25:25,060 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T12:25:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2c29738f993c443c92543d4f4d79e753 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753 2024-11-07T12:25:25,065 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T12:25:25,066 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ea4184a46d62bfe1df974384dfe56d56 in 868ms, sequenceid=192, compaction requested=false 2024-11-07T12:25:25,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:25,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-07T12:25:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-07T12:25:25,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T12:25:25,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6350 sec 2024-11-07T12:25:25,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.6400 sec 2024-11-07T12:25:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:25,125 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:25,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:25:25,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/291d5c7212cd481ca6e8a093e8db6dfb is 50, key is test_row_0/A:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742322_1498 (size=14541) 2024-11-07T12:25:25,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/291d5c7212cd481ca6e8a093e8db6dfb 2024-11-07T12:25:25,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e9d65b847cb74235ba5b2b4d996841f1 is 50, key is test_row_0/B:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742323_1499 (size=12151) 2024-11-07T12:25:25,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e9d65b847cb74235ba5b2b4d996841f1 2024-11-07T12:25:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982385181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982385182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982385183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f is 50, key is test_row_0/C:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742324_1500 (size=12151) 2024-11-07T12:25:25,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f 2024-11-07T12:25:25,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982385196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/291d5c7212cd481ca6e8a093e8db6dfb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb 2024-11-07T12:25:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982385202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb, entries=200, sequenceid=211, filesize=14.2 K 2024-11-07T12:25:25,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/e9d65b847cb74235ba5b2b4d996841f1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1 2024-11-07T12:25:25,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1, entries=150, sequenceid=211, filesize=11.9 K 2024-11-07T12:25:25,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f 2024-11-07T12:25:25,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f, entries=150, sequenceid=211, filesize=11.9 K 2024-11-07T12:25:25,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for ea4184a46d62bfe1df974384dfe56d56 in 108ms, sequenceid=211, compaction requested=true 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:25,233 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:25,233 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:25,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:25,234 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:25,234 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:25,234 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,234 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/ef7244d04edd4e05876de9c29e201a1b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.0 K 2024-11-07T12:25:25,235 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ef7244d04edd4e05876de9c29e201a1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:25,235 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:25,235 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:25,235 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,236 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8af16b19de34576862ed73dc2a92e29, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=38.3 K 2024-11-07T12:25:25,236 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting f72e20359a87488aaa3611bd5c7f3da5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730982323311 2024-11-07T12:25:25,236 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8af16b19de34576862ed73dc2a92e29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:25,236 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d65b847cb74235ba5b2b4d996841f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324491 2024-11-07T12:25:25,236 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8c45c3822e74db29dc00e6f5d894086, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730982323311 2024-11-07T12:25:25,236 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 291d5c7212cd481ca6e8a093e8db6dfb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324490 2024-11-07T12:25:25,246 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:25,247 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/4af0ad5b9a774c23ab88c2a306487b94 is 50, key is test_row_0/B:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,249 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:25,250 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/8a41b9d210f84fa8a5e448899870ab01 is 50, key is test_row_0/A:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742326_1502 (size=12663) 2024-11-07T12:25:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742325_1501 (size=12663) 2024-11-07T12:25:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:25,308 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:25,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:25:25,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/dfadb62cb4b94f3b80a61b54821b252e is 50, key is test_row_0/A:col10/1730982325193/Put/seqid=0 2024-11-07T12:25:25,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742327_1503 (size=14541) 2024-11-07T12:25:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982385368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982385369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982385370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982385379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982385388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982385491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982385491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982385501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982385501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982385501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T12:25:25,581 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:25,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-07T12:25:25,584 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:25,585 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:25,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:25,674 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/8a41b9d210f84fa8a5e448899870ab01 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a41b9d210f84fa8a5e448899870ab01 2024-11-07T12:25:25,674 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/4af0ad5b9a774c23ab88c2a306487b94 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4af0ad5b9a774c23ab88c2a306487b94 2024-11-07T12:25:25,680 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 8a41b9d210f84fa8a5e448899870ab01(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:25,680 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:25,680 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982325233; duration=0sec 2024-11-07T12:25:25,680 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:25,680 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:25,680 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:25,681 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:25,681 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:25,681 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,682 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/09a14c9cd6114398875d66d4dc9d1240, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.0 K 2024-11-07T12:25:25,682 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a14c9cd6114398875d66d4dc9d1240, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730982322132 2024-11-07T12:25:25,682 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c29738f993c443c92543d4f4d79e753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730982323311 2024-11-07T12:25:25,683 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e8cb438b9f4ea5a093e7bdd51b1a7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324491 2024-11-07T12:25:25,690 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into 4af0ad5b9a774c23ab88c2a306487b94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:25,690 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:25,690 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982325233; duration=0sec 2024-11-07T12:25:25,690 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:25,690 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:25,694 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:25,695 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/b5f34b5bb1754dfc82255ed6a438715b is 50, key is test_row_0/C:col10/1730982324491/Put/seqid=0 2024-11-07T12:25:25,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742328_1504 (size=12663) 2024-11-07T12:25:25,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982385701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982385701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982385720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982385720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982385721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:25,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/dfadb62cb4b94f3b80a61b54821b252e 2024-11-07T12:25:25,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:25,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:25,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/77f5ad978f7d4df787591f56afc21402 is 50, key is test_row_0/B:col10/1730982325193/Put/seqid=0 2024-11-07T12:25:25,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742329_1505 (size=12151) 2024-11-07T12:25:25,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:25,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982386030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982386030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982386031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982386031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982386032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,123 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/b5f34b5bb1754dfc82255ed6a438715b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5f34b5bb1754dfc82255ed6a438715b 2024-11-07T12:25:26,134 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into b5f34b5bb1754dfc82255ed6a438715b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:26,134 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:26,134 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982325233; duration=0sec 2024-11-07T12:25:26,134 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:26,134 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:26,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/77f5ad978f7d4df787591f56afc21402 2024-11-07T12:25:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:26,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/3b169e52966b448ca710dc31d55e7c33 is 50, key is test_row_0/C:col10/1730982325193/Put/seqid=0 2024-11-07T12:25:26,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742330_1506 (size=12151) 2024-11-07T12:25:26,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982386541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982386542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:26,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:26,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:26,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/3b169e52966b448ca710dc31d55e7c33 2024-11-07T12:25:26,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/dfadb62cb4b94f3b80a61b54821b252e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e 2024-11-07T12:25:26,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e, entries=200, sequenceid=233, filesize=14.2 K 2024-11-07T12:25:26,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/77f5ad978f7d4df787591f56afc21402 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402 2024-11-07T12:25:26,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402, entries=150, sequenceid=233, filesize=11.9 K 2024-11-07T12:25:26,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/3b169e52966b448ca710dc31d55e7c33 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33 2024-11-07T12:25:26,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33, entries=150, sequenceid=233, filesize=11.9 K 2024-11-07T12:25:26,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ea4184a46d62bfe1df974384dfe56d56 in 1381ms, sequenceid=233, compaction requested=false 2024-11-07T12:25:26,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:26,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T12:25:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:26,818 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:25:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/2147adc2e7864939904ea3c7e1626443 is 50, key is test_row_0/A:col10/1730982325369/Put/seqid=0 2024-11-07T12:25:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742331_1507 (size=12151) 2024-11-07T12:25:26,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/2147adc2e7864939904ea3c7e1626443 2024-11-07T12:25:26,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/034d89f4b24c402c86f821530dc68ebb is 50, key is test_row_0/B:col10/1730982325369/Put/seqid=0 2024-11-07T12:25:26,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742332_1508 (size=12151) 2024-11-07T12:25:27,281 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/034d89f4b24c402c86f821530dc68ebb 2024-11-07T12:25:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/dda494e1c66546aaa2bf988cac29cc5f is 50, key is test_row_0/C:col10/1730982325369/Put/seqid=0 2024-11-07T12:25:27,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742333_1509 (size=12151) 2024-11-07T12:25:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:27,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982387621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982387621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982387621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982387622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982387622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,694 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/dda494e1c66546aaa2bf988cac29cc5f 2024-11-07T12:25:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/2147adc2e7864939904ea3c7e1626443 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443 2024-11-07T12:25:27,700 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T12:25:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/034d89f4b24c402c86f821530dc68ebb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb 2024-11-07T12:25:27,706 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T12:25:27,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/dda494e1c66546aaa2bf988cac29cc5f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f 2024-11-07T12:25:27,710 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T12:25:27,711 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ea4184a46d62bfe1df974384dfe56d56 in 893ms, sequenceid=250, compaction requested=true 2024-11-07T12:25:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-07T12:25:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-07T12:25:27,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T12:25:27,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-07T12:25:27,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1370 sec 2024-11-07T12:25:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T12:25:27,731 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:27,731 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T12:25:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:27,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:27,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:27,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:27,747 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:27,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T12:25:27,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/24e912fb542a430dba52c149cea31225 is 50, key is test_row_0/A:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:27,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742334_1510 (size=17181) 2024-11-07T12:25:27,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982387751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982387759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982387761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982387761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982387762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982387871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982387871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982387872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982387872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982387872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:27,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:27,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:28,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982388081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982388092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982388092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982388092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982388092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/24e912fb542a430dba52c149cea31225 2024-11-07T12:25:28,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/af2708a831954be3a892efcc4c9e3757 is 50, key is test_row_0/B:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:28,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742335_1511 (size=12301) 2024-11-07T12:25:28,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/af2708a831954be3a892efcc4c9e3757 2024-11-07T12:25:28,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/6a1479c6772344cab02b628df5a30d46 is 50, key is test_row_0/C:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:28,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742336_1512 (size=12301) 2024-11-07T12:25:28,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:28,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982388401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982388411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:28,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:28,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/6a1479c6772344cab02b628df5a30d46 2024-11-07T12:25:28,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/24e912fb542a430dba52c149cea31225 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225 2024-11-07T12:25:28,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225, entries=250, sequenceid=275, filesize=16.8 K 2024-11-07T12:25:28,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/af2708a831954be3a892efcc4c9e3757 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757 2024-11-07T12:25:28,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T12:25:28,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/6a1479c6772344cab02b628df5a30d46 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46 2024-11-07T12:25:28,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T12:25:28,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ea4184a46d62bfe1df974384dfe56d56 in 880ms, sequenceid=275, compaction requested=true 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:28,628 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:25:28,628 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:28,629 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56536 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:25:28,629 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:28,629 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,629 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a41b9d210f84fa8a5e448899870ab01, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=55.2 K 2024-11-07T12:25:28,629 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:25:28,629 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a41b9d210f84fa8a5e448899870ab01, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324491 2024-11-07T12:25:28,629 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:28,629 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,629 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4af0ad5b9a774c23ab88c2a306487b94, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=48.1 K 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfadb62cb4b94f3b80a61b54821b252e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730982325178 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af0ad5b9a774c23ab88c2a306487b94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324491 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2147adc2e7864939904ea3c7e1626443, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982325369 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f5ad978f7d4df787591f56afc21402, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730982325193 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e912fb542a430dba52c149cea31225, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327581 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 034d89f4b24c402c86f821530dc68ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982325369 2024-11-07T12:25:28,630 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting af2708a831954be3a892efcc4c9e3757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327620 2024-11-07T12:25:28,637 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:28,637 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/371524c1b665422296475613241f176b is 50, key is test_row_0/B:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:28,638 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#430 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:28,638 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ba9149221da24def9d307c4ba662c791 is 50, key is test_row_0/A:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:28,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742337_1513 (size=12949) 2024-11-07T12:25:28,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742338_1514 (size=12949) 2024-11-07T12:25:28,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T12:25:28,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:28,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:25:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/1848fec81dc3414a8788495bf931051b is 50, key is test_row_0/A:col10/1730982327750/Put/seqid=0 2024-11-07T12:25:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742339_1515 (size=12301) 2024-11-07T12:25:28,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/1848fec81dc3414a8788495bf931051b 2024-11-07T12:25:28,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/453ffe6a6db346f0a63550693b94affe is 50, key is test_row_0/B:col10/1730982327750/Put/seqid=0 2024-11-07T12:25:28,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742340_1516 (size=12301) 2024-11-07T12:25:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:28,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982389008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982389009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982389009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982389011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982389023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,048 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/371524c1b665422296475613241f176b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/371524c1b665422296475613241f176b 2024-11-07T12:25:29,054 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into 371524c1b665422296475613241f176b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:29,054 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:29,054 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=12, startTime=1730982328628; duration=0sec 2024-11-07T12:25:29,054 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:29,054 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:29,054 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:25:29,054 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ba9149221da24def9d307c4ba662c791 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba9149221da24def9d307c4ba662c791 2024-11-07T12:25:29,055 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:25:29,055 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:29,055 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:29,055 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5f34b5bb1754dfc82255ed6a438715b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=48.1 K 2024-11-07T12:25:29,056 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b5f34b5bb1754dfc82255ed6a438715b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730982324491 2024-11-07T12:25:29,056 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b169e52966b448ca710dc31d55e7c33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1730982325193 2024-11-07T12:25:29,057 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting dda494e1c66546aaa2bf988cac29cc5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730982325369 2024-11-07T12:25:29,057 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a1479c6772344cab02b628df5a30d46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327620 2024-11-07T12:25:29,061 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into ba9149221da24def9d307c4ba662c791(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:29,061 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:29,061 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=12, startTime=1730982328628; duration=0sec 2024-11-07T12:25:29,061 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:29,061 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:29,070 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:29,071 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/73fb949f482940b4ab746cceac8acc79 is 50, key is test_row_0/C:col10/1730982327746/Put/seqid=0 2024-11-07T12:25:29,077 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/453ffe6a6db346f0a63550693b94affe 2024-11-07T12:25:29,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742341_1517 (size=12949) 2024-11-07T12:25:29,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2a0885fa40364fabb764768420683508 is 50, key is test_row_0/C:col10/1730982327750/Put/seqid=0 2024-11-07T12:25:29,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742342_1518 (size=12301) 2024-11-07T12:25:29,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2a0885fa40364fabb764768420683508 2024-11-07T12:25:29,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/1848fec81dc3414a8788495bf931051b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b 2024-11-07T12:25:29,128 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b, entries=150, sequenceid=286, filesize=12.0 K 2024-11-07T12:25:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/453ffe6a6db346f0a63550693b94affe as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe 2024-11-07T12:25:29,132 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe, entries=150, sequenceid=286, filesize=12.0 K 2024-11-07T12:25:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2a0885fa40364fabb764768420683508 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508 2024-11-07T12:25:29,136 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508, entries=150, sequenceid=286, filesize=12.0 K 2024-11-07T12:25:29,137 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ea4184a46d62bfe1df974384dfe56d56 in 482ms, sequenceid=286, compaction requested=false 2024-11-07T12:25:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T12:25:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T12:25:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:29,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T12:25:29,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-07T12:25:29,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T12:25:29,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4100 sec 2024-11-07T12:25:29,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/384eaabf78494925a228459b16dfe5e6 is 50, key is test_row_0/A:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:29,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742343_1519 (size=14741) 2024-11-07T12:25:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982389144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982389145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982389146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982389156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982389157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982389261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982389261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982389261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982389261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982389281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982389471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982389480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982389481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982389481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,500 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/73fb949f482940b4ab746cceac8acc79 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/73fb949f482940b4ab746cceac8acc79 2024-11-07T12:25:29,505 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into 73fb949f482940b4ab746cceac8acc79(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:29,505 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:29,505 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=12, startTime=1730982328628; duration=0sec 2024-11-07T12:25:29,505 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:29,505 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982389504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/384eaabf78494925a228459b16dfe5e6 2024-11-07T12:25:29,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/215e78ecbae44293a80f10255f7b2dda is 50, key is test_row_0/B:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:29,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742344_1520 (size=12301) 2024-11-07T12:25:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982389791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982389791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982389791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982389791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982389821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T12:25:29,881 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:29,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T12:25:29,883 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:29,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:29,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:29,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/215e78ecbae44293a80f10255f7b2dda 2024-11-07T12:25:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:29,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/b5a39bb5d89a43ac9350510e8aeeccb2 is 50, key is test_row_0/C:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742345_1521 (size=12301) 2024-11-07T12:25:30,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/b5a39bb5d89a43ac9350510e8aeeccb2 2024-11-07T12:25:30,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/384eaabf78494925a228459b16dfe5e6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6 2024-11-07T12:25:30,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6, entries=200, sequenceid=314, filesize=14.4 K 2024-11-07T12:25:30,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/215e78ecbae44293a80f10255f7b2dda as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda 2024-11-07T12:25:30,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda, entries=150, sequenceid=314, filesize=12.0 K 2024-11-07T12:25:30,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/b5a39bb5d89a43ac9350510e8aeeccb2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2 2024-11-07T12:25:30,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2, entries=150, sequenceid=314, filesize=12.0 K 2024-11-07T12:25:30,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:25:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:30,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ea4184a46d62bfe1df974384dfe56d56 in 897ms, sequenceid=314, compaction requested=true 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:30,038 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T12:25:30,038 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:30,039 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:30,039 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:30,039 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,039 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:30,039 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/371524c1b665422296475613241f176b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.7 K 2024-11-07T12:25:30,039 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:30,040 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,040 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba9149221da24def9d307c4ba662c791, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=39.1 K 2024-11-07T12:25:30,040 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 371524c1b665422296475613241f176b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327620 2024-11-07T12:25:30,040 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba9149221da24def9d307c4ba662c791, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327620 2024-11-07T12:25:30,040 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 453ffe6a6db346f0a63550693b94affe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982327750 2024-11-07T12:25:30,040 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1848fec81dc3414a8788495bf931051b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982327750 2024-11-07T12:25:30,041 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 215e78ecbae44293a80f10255f7b2dda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:30,041 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384eaabf78494925a228459b16dfe5e6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:30,070 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:30,071 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/94ed05a408df4343b016355be52e6ae2 is 50, key is test_row_0/B:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:30,087 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:30,087 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/3d0eee85ed5647dda667e6ec4da1a0c6 is 50, key is test_row_0/A:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:30,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742346_1522 (size=13051) 2024-11-07T12:25:30,117 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/94ed05a408df4343b016355be52e6ae2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/94ed05a408df4343b016355be52e6ae2 2024-11-07T12:25:30,121 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into 94ed05a408df4343b016355be52e6ae2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:30,121 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:30,121 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982330038; duration=0sec 2024-11-07T12:25:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:30,122 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:30,123 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,123 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/73fb949f482940b4ab746cceac8acc79, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.7 K 2024-11-07T12:25:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fb949f482940b4ab746cceac8acc79, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730982327620 2024-11-07T12:25:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0885fa40364fabb764768420683508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1730982327750 2024-11-07T12:25:30,123 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a39bb5d89a43ac9350510e8aeeccb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742347_1523 (size=13051) 2024-11-07T12:25:30,142 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:30,142 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ee849f44e56049279769a60c0f4eef8c is 50, key is test_row_0/C:col10/1730982329006/Put/seqid=0 2024-11-07T12:25:30,151 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/3d0eee85ed5647dda667e6ec4da1a0c6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3d0eee85ed5647dda667e6ec4da1a0c6 2024-11-07T12:25:30,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742348_1524 (size=13051) 2024-11-07T12:25:30,161 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 3d0eee85ed5647dda667e6ec4da1a0c6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:30,162 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:30,162 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982330038; duration=0sec 2024-11-07T12:25:30,162 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:30,162 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:30,164 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ee849f44e56049279769a60c0f4eef8c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ee849f44e56049279769a60c0f4eef8c 2024-11-07T12:25:30,168 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into ee849f44e56049279769a60c0f4eef8c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:30,168 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:30,168 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982330038; duration=0sec 2024-11-07T12:25:30,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:30,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:30,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T12:25:30,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:30,193 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T12:25:30,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/88c42fb834bb4780be7be3f43ee698a1 is 50, key is test_row_0/A:col10/1730982329144/Put/seqid=0 2024-11-07T12:25:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742349_1525 (size=12301) 2024-11-07T12:25:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:30,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982390338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982390339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982390344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982390345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982390349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982390451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982390460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982390460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982390460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982390461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:25:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:30,658 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/88c42fb834bb4780be7be3f43ee698a1 2024-11-07T12:25:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/aa9f0f841b184ec2a18640c8cca8202e is 50, key is test_row_0/B:col10/1730982329144/Put/seqid=0 2024-11-07T12:25:30,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742350_1526 (size=12301) 2024-11-07T12:25:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982390671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982390981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982390981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982390981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982390981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982390981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:31,068 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/aa9f0f841b184ec2a18640c8cca8202e 2024-11-07T12:25:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/867755e76e7b4f569f3cf3304cb639c7 is 50, key is test_row_0/C:col10/1730982329144/Put/seqid=0 2024-11-07T12:25:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742351_1527 (size=12301) 2024-11-07T12:25:31,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/867755e76e7b4f569f3cf3304cb639c7 2024-11-07T12:25:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/88c42fb834bb4780be7be3f43ee698a1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1 2024-11-07T12:25:31,489 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:25:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/aa9f0f841b184ec2a18640c8cca8202e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e 2024-11-07T12:25:31,492 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:25:31,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/867755e76e7b4f569f3cf3304cb639c7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7 2024-11-07T12:25:31,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982391490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982391491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T12:25:31,498 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ea4184a46d62bfe1df974384dfe56d56 in 1306ms, sequenceid=328, compaction requested=false 2024-11-07T12:25:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T12:25:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T12:25:31,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T12:25:31,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-07T12:25:31,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6210 sec 2024-11-07T12:25:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:31,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:31,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T12:25:31,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/9160c7d27c564a178b0d2054e6c4ec35 is 50, key is test_row_0/A:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:31,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742352_1528 (size=14741) 2024-11-07T12:25:31,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982391521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982391522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982391532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982391641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982391641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982391650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982391850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982391850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982391871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:31,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/9160c7d27c564a178b0d2054e6c4ec35 2024-11-07T12:25:31,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/0316548671a6450bbd8a9753f75521ea is 50, key is test_row_0/B:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:31,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742353_1529 (size=12301) 2024-11-07T12:25:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T12:25:32,041 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:32,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T12:25:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:32,043 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:32,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:32,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982392161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982392161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982392180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:25:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/0316548671a6450bbd8a9753f75521ea 2024-11-07T12:25:32,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/1da538cc68764e06a0cd84b02ba78b80 is 50, key is test_row_0/C:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:32,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:25:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:32,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742354_1530 (size=12301) 2024-11-07T12:25:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:32,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:25:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982392501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,508 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in ea4184a46d62bfe1df974384dfe56d56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982392511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,518 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in ea4184a46d62bfe1df974384dfe56d56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:25:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982392670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982392681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982392701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:32,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/1da538cc68764e06a0cd84b02ba78b80 2024-11-07T12:25:32,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/9160c7d27c564a178b0d2054e6c4ec35 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35 2024-11-07T12:25:32,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35, entries=200, sequenceid=355, filesize=14.4 K 2024-11-07T12:25:32,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/0316548671a6450bbd8a9753f75521ea as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea 2024-11-07T12:25:32,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T12:25:32,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/1da538cc68764e06a0cd84b02ba78b80 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80 2024-11-07T12:25:32,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T12:25:32,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ea4184a46d62bfe1df974384dfe56d56 in 1264ms, sequenceid=355, compaction requested=true 2024-11-07T12:25:32,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:32,770 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:32,770 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:32,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:32,771 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:32,771 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:32,771 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,771 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3d0eee85ed5647dda667e6ec4da1a0c6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=39.2 K 2024-11-07T12:25:32,771 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0eee85ed5647dda667e6ec4da1a0c6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:32,772 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,772 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/94ed05a408df4343b016355be52e6ae2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.8 K 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88c42fb834bb4780be7be3f43ee698a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730982329141 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ed05a408df4343b016355be52e6ae2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9160c7d27c564a178b0d2054e6c4ec35, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9f0f841b184ec2a18640c8cca8202e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730982329141 2024-11-07T12:25:32,772 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 0316548671a6450bbd8a9753f75521ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:32,780 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:32,780 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/166cd2dacbcf45bfbc62b41c3a2621c4 is 50, key is test_row_0/A:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:32,784 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:32,784 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/077408f83e1a4c6f85db313cc5966106 is 50, key is test_row_0/B:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742355_1531 (size=13153) 2024-11-07T12:25:32,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742356_1532 (size=13153) 2024-11-07T12:25:32,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T12:25:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:32,807 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T12:25:32,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/527ec5162cc04d998d44e80254acefaa is 50, key is test_row_0/A:col10/1730982331520/Put/seqid=0 2024-11-07T12:25:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742357_1533 (size=12301) 2024-11-07T12:25:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:33,189 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/166cd2dacbcf45bfbc62b41c3a2621c4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/166cd2dacbcf45bfbc62b41c3a2621c4 2024-11-07T12:25:33,194 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/077408f83e1a4c6f85db313cc5966106 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/077408f83e1a4c6f85db313cc5966106 2024-11-07T12:25:33,194 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 166cd2dacbcf45bfbc62b41c3a2621c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:33,194 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:33,194 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982332769; duration=0sec 2024-11-07T12:25:33,194 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:33,194 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:33,194 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:33,195 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:33,196 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:33,196 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:33,196 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ee849f44e56049279769a60c0f4eef8c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.8 K 2024-11-07T12:25:33,196 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee849f44e56049279769a60c0f4eef8c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1730982329006 2024-11-07T12:25:33,196 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867755e76e7b4f569f3cf3304cb639c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730982329141 2024-11-07T12:25:33,197 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da538cc68764e06a0cd84b02ba78b80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:33,198 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into 077408f83e1a4c6f85db313cc5966106(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:33,198 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:33,199 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982332770; duration=0sec 2024-11-07T12:25:33,199 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:33,199 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:33,203 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:33,204 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/facfff3a52244d9ea055220095a04212 is 50, key is test_row_0/C:col10/1730982330347/Put/seqid=0 2024-11-07T12:25:33,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742358_1534 (size=13153) 2024-11-07T12:25:33,213 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/facfff3a52244d9ea055220095a04212 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/facfff3a52244d9ea055220095a04212 2024-11-07T12:25:33,216 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into facfff3a52244d9ea055220095a04212(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:33,216 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:33,216 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982332770; duration=0sec 2024-11-07T12:25:33,216 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:33,216 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:33,224 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/527ec5162cc04d998d44e80254acefaa 2024-11-07T12:25:33,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/8049e88e903b470294187137577e89c7 is 50, key is test_row_0/B:col10/1730982331520/Put/seqid=0 2024-11-07T12:25:33,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742359_1535 (size=12301) 2024-11-07T12:25:33,241 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/8049e88e903b470294187137577e89c7 2024-11-07T12:25:33,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e35d606d1c03413aadff287f33bf4f87 is 50, key is test_row_0/C:col10/1730982331520/Put/seqid=0 2024-11-07T12:25:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742360_1536 (size=12301) 2024-11-07T12:25:33,653 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e35d606d1c03413aadff287f33bf4f87 2024-11-07T12:25:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/527ec5162cc04d998d44e80254acefaa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa 2024-11-07T12:25:33,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:25:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/8049e88e903b470294187137577e89c7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7 2024-11-07T12:25:33,668 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:25:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e35d606d1c03413aadff287f33bf4f87 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87 2024-11-07T12:25:33,673 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87, entries=150, sequenceid=364, filesize=12.0 K 2024-11-07T12:25:33,674 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for ea4184a46d62bfe1df974384dfe56d56 in 866ms, sequenceid=364, compaction requested=false 2024-11-07T12:25:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T12:25:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T12:25:33,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T12:25:33,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6310 sec 2024-11-07T12:25:33,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.6360 sec 2024-11-07T12:25:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:33,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T12:25:33,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/3fe6238624ed43e09f032c0817992648 is 50, key is test_row_0/A:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:33,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742361_1537 (size=17181) 2024-11-07T12:25:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982393807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982393808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:33,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982393809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:33,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982393921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982393921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982393921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/3fe6238624ed43e09f032c0817992648 2024-11-07T12:25:34,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/b1fb66b397734437b6239661839a7562 is 50, key is test_row_0/B:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742362_1538 (size=12301) 2024-11-07T12:25:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982394131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982394131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982394131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T12:25:34,191 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:34,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T12:25:34,193 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:34,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:34,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:34,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:25:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1730982394442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53308 deadline: 1730982394442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53338 deadline: 1730982394442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:25:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:34,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/b1fb66b397734437b6239661839a7562 2024-11-07T12:25:34,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ef85026fe1ce461d8c7edda3062d8609 is 50, key is test_row_0/C:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1730982394530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742363_1539 (size=12301) 2024-11-07T12:25:34,533 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in ea4184a46d62bfe1df974384dfe56d56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53280 deadline: 1730982394540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:34,542 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in ea4184a46d62bfe1df974384dfe56d56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 , the old value is region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., hostname=8944cbb06cd2,46221,1730982182218, seqNum=2 because the exception is null or not the one we care about 2024-11-07T12:25:34,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:25:34,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:34,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:25:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:34,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ef85026fe1ce461d8c7edda3062d8609 2024-11-07T12:25:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/3fe6238624ed43e09f032c0817992648 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648 2024-11-07T12:25:34,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648, entries=250, sequenceid=378, filesize=16.8 K 2024-11-07T12:25:34,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/b1fb66b397734437b6239661839a7562 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562 2024-11-07T12:25:34,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562, entries=150, sequenceid=378, filesize=12.0 K 2024-11-07T12:25:34,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/ef85026fe1ce461d8c7edda3062d8609 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609 2024-11-07T12:25:34,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609, entries=150, sequenceid=378, filesize=12.0 K 2024-11-07T12:25:34,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ea4184a46d62bfe1df974384dfe56d56 in 1255ms, sequenceid=378, compaction requested=true 2024-11-07T12:25:34,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:34,959 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:34,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:34,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:34,959 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:34,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T12:25:34,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,959 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T12:25:34,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:34,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:34,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ea4184a46d62bfe1df974384dfe56d56:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:34,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:34,960 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:34,960 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/A is initiating minor compaction (all files) 2024-11-07T12:25:34,960 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/A in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,960 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/166cd2dacbcf45bfbc62b41c3a2621c4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=41.6 K 2024-11-07T12:25:34,962 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 166cd2dacbcf45bfbc62b41c3a2621c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:34,962 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:34,962 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/B is initiating minor compaction (all files) 2024-11-07T12:25:34,962 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/B in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:34,962 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/077408f83e1a4c6f85db313cc5966106, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.9 K 2024-11-07T12:25:34,968 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 527ec5162cc04d998d44e80254acefaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730982331516 2024-11-07T12:25:34,968 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077408f83e1a4c6f85db313cc5966106, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:34,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. as already flushing 2024-11-07T12:25:34,969 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe6238624ed43e09f032c0817992648, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1730982333694 2024-11-07T12:25:34,969 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8049e88e903b470294187137577e89c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730982331516 2024-11-07T12:25:34,969 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-11-07T12:25:34,969 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,969 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,969 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,970 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fb66b397734437b6239661839a7562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1730982333694 2024-11-07T12:25:34,970 INFO [Thread-2040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2040. 2024-11-07T12:25:34,970 DEBUG [Thread-2040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,970 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,970 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,971 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-11-07T12:25:34,971 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,971 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,972 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,972 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-11-07T12:25:34,972 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,972 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,972 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,974 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-11-07T12:25:34,974 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,974 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,974 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ba8f9f23219a4f108c577ceef78529a9 is 50, key is test_row_0/A:col10/1730982333806/Put/seqid=0 2024-11-07T12:25:34,980 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#A#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:34,981 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/8a552b15055c4d369b3d314e5d209348 is 50, key is test_row_0/A:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:34,983 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-11-07T12:25:34,983 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,983 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,983 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,984 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-07T12:25:34,984 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,984 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,984 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,986 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:34,987 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/a78f9529e8ad42e8b37460b70cc0bd62 is 50, key is test_row_0/B:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:34,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,988 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-07T12:25:34,988 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:34,988 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,988 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:34,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:34,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742364_1540 (size=12301) 2024-11-07T12:25:34,993 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ba8f9f23219a4f108c577ceef78529a9 2024-11-07T12:25:35,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f27d288b576b4bc2a325d100810a30b2 is 50, key is test_row_0/B:col10/1730982333806/Put/seqid=0 2024-11-07T12:25:35,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742365_1541 (size=13255) 2024-11-07T12:25:35,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742366_1542 (size=13255) 2024-11-07T12:25:35,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742367_1543 (size=12301) 2024-11-07T12:25:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:35,413 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f27d288b576b4bc2a325d100810a30b2 2024-11-07T12:25:35,415 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/a78f9529e8ad42e8b37460b70cc0bd62 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/a78f9529e8ad42e8b37460b70cc0bd62 2024-11-07T12:25:35,415 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/8a552b15055c4d369b3d314e5d209348 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a552b15055c4d369b3d314e5d209348 2024-11-07T12:25:35,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d03d08f6f0494ef6b1f86a00c98a77b7 is 50, key is test_row_0/C:col10/1730982333806/Put/seqid=0 2024-11-07T12:25:35,421 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/B of ea4184a46d62bfe1df974384dfe56d56 into a78f9529e8ad42e8b37460b70cc0bd62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:35,421 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/B, priority=13, startTime=1730982334959; duration=0sec 2024-11-07T12:25:35,421 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/A of ea4184a46d62bfe1df974384dfe56d56 into 8a552b15055c4d369b3d314e5d209348(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:35,421 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/A, priority=13, startTime=1730982334959; duration=0sec 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:B 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:35,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:A 2024-11-07T12:25:35,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742368_1544 (size=12301) 2024-11-07T12:25:35,422 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:35,422 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): ea4184a46d62bfe1df974384dfe56d56/C is initiating minor compaction (all files) 2024-11-07T12:25:35,422 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ea4184a46d62bfe1df974384dfe56d56/C in TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:35,422 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/facfff3a52244d9ea055220095a04212, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp, totalSize=36.9 K 2024-11-07T12:25:35,423 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting facfff3a52244d9ea055220095a04212, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730982330335 2024-11-07T12:25:35,423 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35d606d1c03413aadff287f33bf4f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730982331516 2024-11-07T12:25:35,423 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef85026fe1ce461d8c7edda3062d8609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1730982333694 2024-11-07T12:25:35,427 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ea4184a46d62bfe1df974384dfe56d56#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:35,427 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2bb240e3897e4d979cb8ea96983db7ad is 50, key is test_row_0/C:col10/1730982333694/Put/seqid=0 2024-11-07T12:25:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742369_1545 (size=13255) 2024-11-07T12:25:35,823 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d03d08f6f0494ef6b1f86a00c98a77b7 2024-11-07T12:25:35,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ba8f9f23219a4f108c577ceef78529a9 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba8f9f23219a4f108c577ceef78529a9 2024-11-07T12:25:35,830 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba8f9f23219a4f108c577ceef78529a9, entries=150, sequenceid=403, filesize=12.0 K 2024-11-07T12:25:35,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/f27d288b576b4bc2a325d100810a30b2 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f27d288b576b4bc2a325d100810a30b2 2024-11-07T12:25:35,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f27d288b576b4bc2a325d100810a30b2, entries=150, sequenceid=403, filesize=12.0 K 2024-11-07T12:25:35,834 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/2bb240e3897e4d979cb8ea96983db7ad as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2bb240e3897e4d979cb8ea96983db7ad 2024-11-07T12:25:35,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/d03d08f6f0494ef6b1f86a00c98a77b7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d03d08f6f0494ef6b1f86a00c98a77b7 2024-11-07T12:25:35,837 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d03d08f6f0494ef6b1f86a00c98a77b7, entries=150, sequenceid=403, filesize=12.0 K 2024-11-07T12:25:35,838 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=20.13 KB/20610 for ea4184a46d62bfe1df974384dfe56d56 in 879ms, sequenceid=403, compaction requested=false 2024-11-07T12:25:35,838 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ea4184a46d62bfe1df974384dfe56d56/C of ea4184a46d62bfe1df974384dfe56d56 into 2bb240e3897e4d979cb8ea96983db7ad(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:35,838 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:35,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for ea4184a46d62bfe1df974384dfe56d56: 2024-11-07T12:25:35,838 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., storeName=ea4184a46d62bfe1df974384dfe56d56/C, priority=13, startTime=1730982334960; duration=0sec 2024-11-07T12:25:35,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:35,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T12:25:35,838 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:35,838 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ea4184a46d62bfe1df974384dfe56d56:C 2024-11-07T12:25:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T12:25:35,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T12:25:35,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-07T12:25:35,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6490 sec 2024-11-07T12:25:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T12:25:36,341 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:38,574 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-11-07T12:25:38,574 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:38,574 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,574 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:38,584 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-11-07T12:25:38,584 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:25:38,585 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,585 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1362 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4081 rows 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1355 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4063 rows 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1359 2024-11-07T12:25:38,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4077 rows 2024-11-07T12:25:38,586 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1365 2024-11-07T12:25:38,586 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4095 rows 2024-11-07T12:25:38,586 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1373 2024-11-07T12:25:38,586 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4119 rows 2024-11-07T12:25:38,586 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:25:38,586 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:25:38,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:38,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:25:38,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:25:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:38,594 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982338593"}]},"ts":"1730982338593"} 2024-11-07T12:25:38,595 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:25:38,595 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:25:38,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:25:38,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, UNASSIGN}] 2024-11-07T12:25:38,598 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, UNASSIGN 2024-11-07T12:25:38,599 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=ea4184a46d62bfe1df974384dfe56d56, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:38,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, UNASSIGN because future has completed 2024-11-07T12:25:38,600 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:25:38,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:38,752 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:38,752 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:25:38,752 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing ea4184a46d62bfe1df974384dfe56d56, disabling compactions & flushes 2024-11-07T12:25:38,752 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:38,752 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:38,752 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. after waiting 0 ms 2024-11-07T12:25:38,752 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:38,753 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing ea4184a46d62bfe1df974384dfe56d56 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T12:25:38,757 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ceab24a038af45178f67a38f04b48e04 is 50, key is test_row_0/A:col10/1730982334964/Put/seqid=0 2024-11-07T12:25:38,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742370_1546 (size=12301) 2024-11-07T12:25:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:39,161 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ceab24a038af45178f67a38f04b48e04 2024-11-07T12:25:39,172 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/228b40ad12424942bd85b0a8c36dbb1d is 50, key is test_row_0/B:col10/1730982334964/Put/seqid=0 2024-11-07T12:25:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742371_1547 (size=12301) 2024-11-07T12:25:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:39,576 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/228b40ad12424942bd85b0a8c36dbb1d 2024-11-07T12:25:39,581 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e34591fff50a4249acf78bc0a012f8ae is 50, key is test_row_0/C:col10/1730982334964/Put/seqid=0 2024-11-07T12:25:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742372_1548 (size=12301) 2024-11-07T12:25:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:39,984 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e34591fff50a4249acf78bc0a012f8ae 2024-11-07T12:25:39,989 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/A/ceab24a038af45178f67a38f04b48e04 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ceab24a038af45178f67a38f04b48e04 2024-11-07T12:25:39,993 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ceab24a038af45178f67a38f04b48e04, entries=150, sequenceid=414, filesize=12.0 K 2024-11-07T12:25:39,994 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/B/228b40ad12424942bd85b0a8c36dbb1d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/228b40ad12424942bd85b0a8c36dbb1d 2024-11-07T12:25:39,998 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/228b40ad12424942bd85b0a8c36dbb1d, entries=150, sequenceid=414, filesize=12.0 K 2024-11-07T12:25:39,998 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/.tmp/C/e34591fff50a4249acf78bc0a012f8ae as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e34591fff50a4249acf78bc0a012f8ae 2024-11-07T12:25:40,001 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e34591fff50a4249acf78bc0a012f8ae, entries=150, sequenceid=414, filesize=12.0 K 2024-11-07T12:25:40,002 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ea4184a46d62bfe1df974384dfe56d56 in 1250ms, sequenceid=414, compaction requested=true 2024-11-07T12:25:40,003 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/e33cc6505d024484ad4d6a0acccd3ad2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/000a67f55ca14c4292ef8ab538d3e29c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/382b4e25ef00412c85012e9654364353, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8af16b19de34576862ed73dc2a92e29, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a41b9d210f84fa8a5e448899870ab01, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba9149221da24def9d307c4ba662c791, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3d0eee85ed5647dda667e6ec4da1a0c6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/166cd2dacbcf45bfbc62b41c3a2621c4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648] to archive 2024-11-07T12:25:40,004 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:40,005 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8b78ea1718d4dd2972879d3d5ccef59 2024-11-07T12:25:40,006 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/59c98416736c4414871e9baf2f77a0f4 2024-11-07T12:25:40,008 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/41a9801dc7dc42878b090225971bf892 2024-11-07T12:25:40,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/e33cc6505d024484ad4d6a0acccd3ad2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/e33cc6505d024484ad4d6a0acccd3ad2 2024-11-07T12:25:40,010 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/d350ce4d8e8649919682054b866c6f54 2024-11-07T12:25:40,012 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/05c886c0c49c4ad0bb0ab45510ef1312 2024-11-07T12:25:40,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/000a67f55ca14c4292ef8ab538d3e29c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/000a67f55ca14c4292ef8ab538d3e29c 2024-11-07T12:25:40,014 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/b64d2a56a944473987a40642f625b48b 2024-11-07T12:25:40,016 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/382b4e25ef00412c85012e9654364353 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/382b4e25ef00412c85012e9654364353 2024-11-07T12:25:40,017 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f891816cc13a47b9867f2b62eee6023f 2024-11-07T12:25:40,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/5c1c827f62ef44aa8604bb7503bc3306 2024-11-07T12:25:40,019 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/6f7f4d1936c44c8aa9818f0752bba767 2024-11-07T12:25:40,020 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8af16b19de34576862ed73dc2a92e29 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/c8af16b19de34576862ed73dc2a92e29 2024-11-07T12:25:40,022 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/f8c45c3822e74db29dc00e6f5d894086 2024-11-07T12:25:40,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/291d5c7212cd481ca6e8a093e8db6dfb 2024-11-07T12:25:40,024 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a41b9d210f84fa8a5e448899870ab01 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a41b9d210f84fa8a5e448899870ab01 2024-11-07T12:25:40,025 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/dfadb62cb4b94f3b80a61b54821b252e 2024-11-07T12:25:40,027 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/2147adc2e7864939904ea3c7e1626443 2024-11-07T12:25:40,028 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/24e912fb542a430dba52c149cea31225 2024-11-07T12:25:40,029 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba9149221da24def9d307c4ba662c791 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba9149221da24def9d307c4ba662c791 2024-11-07T12:25:40,031 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/1848fec81dc3414a8788495bf931051b 2024-11-07T12:25:40,032 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/384eaabf78494925a228459b16dfe5e6 2024-11-07T12:25:40,034 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3d0eee85ed5647dda667e6ec4da1a0c6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3d0eee85ed5647dda667e6ec4da1a0c6 2024-11-07T12:25:40,035 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/88c42fb834bb4780be7be3f43ee698a1 2024-11-07T12:25:40,037 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/9160c7d27c564a178b0d2054e6c4ec35 2024-11-07T12:25:40,038 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/166cd2dacbcf45bfbc62b41c3a2621c4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/166cd2dacbcf45bfbc62b41c3a2621c4 2024-11-07T12:25:40,040 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/527ec5162cc04d998d44e80254acefaa 2024-11-07T12:25:40,041 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/3fe6238624ed43e09f032c0817992648 2024-11-07T12:25:40,043 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/bf6b08da7245435aa22e146250baa11c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/cd9d7f6822e147cfb3b9914efb7220d5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/eee1a8a9eaa4479ead834ab65bde2c3f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/ef7244d04edd4e05876de9c29e201a1b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4af0ad5b9a774c23ab88c2a306487b94, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/371524c1b665422296475613241f176b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/94ed05a408df4343b016355be52e6ae2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/077408f83e1a4c6f85db313cc5966106, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562] to archive 2024-11-07T12:25:40,044 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:40,046 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/2d3bb1080b1c4b589f33aaf45c02310b 2024-11-07T12:25:40,048 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/577bf2111ba04a12a5f8483bcf56fc25 2024-11-07T12:25:40,050 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/bf6b08da7245435aa22e146250baa11c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/bf6b08da7245435aa22e146250baa11c 2024-11-07T12:25:40,052 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f5eaa756a1cf4a03ae5dce9e1631c2d0 2024-11-07T12:25:40,053 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/887b1e46fb474649bcd4b03c3fef66ba 2024-11-07T12:25:40,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/cd9d7f6822e147cfb3b9914efb7220d5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/cd9d7f6822e147cfb3b9914efb7220d5 2024-11-07T12:25:40,056 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4987438bd30f4b4c8bd1ace6746bb572 2024-11-07T12:25:40,057 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/6106970c9bea49d09fac1aae4c48faba 2024-11-07T12:25:40,058 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/eee1a8a9eaa4479ead834ab65bde2c3f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/eee1a8a9eaa4479ead834ab65bde2c3f 2024-11-07T12:25:40,060 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/45cbf8203cd046bfbc2453f210687fe1 2024-11-07T12:25:40,061 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e88b4e31a61e42689be34285f8d4b0b9 2024-11-07T12:25:40,062 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/ef7244d04edd4e05876de9c29e201a1b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/ef7244d04edd4e05876de9c29e201a1b 2024-11-07T12:25:40,063 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/c28c45785a474fd48fcbcdd4bc3221f4 2024-11-07T12:25:40,064 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f72e20359a87488aaa3611bd5c7f3da5 2024-11-07T12:25:40,066 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4af0ad5b9a774c23ab88c2a306487b94 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/4af0ad5b9a774c23ab88c2a306487b94 2024-11-07T12:25:40,067 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/e9d65b847cb74235ba5b2b4d996841f1 2024-11-07T12:25:40,069 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/77f5ad978f7d4df787591f56afc21402 2024-11-07T12:25:40,070 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/034d89f4b24c402c86f821530dc68ebb 2024-11-07T12:25:40,071 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/371524c1b665422296475613241f176b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/371524c1b665422296475613241f176b 2024-11-07T12:25:40,072 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/af2708a831954be3a892efcc4c9e3757 2024-11-07T12:25:40,074 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/453ffe6a6db346f0a63550693b94affe 2024-11-07T12:25:40,075 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/94ed05a408df4343b016355be52e6ae2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/94ed05a408df4343b016355be52e6ae2 2024-11-07T12:25:40,076 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/215e78ecbae44293a80f10255f7b2dda 2024-11-07T12:25:40,078 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/aa9f0f841b184ec2a18640c8cca8202e 2024-11-07T12:25:40,079 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/077408f83e1a4c6f85db313cc5966106 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/077408f83e1a4c6f85db313cc5966106 2024-11-07T12:25:40,080 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/0316548671a6450bbd8a9753f75521ea 2024-11-07T12:25:40,081 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/8049e88e903b470294187137577e89c7 2024-11-07T12:25:40,082 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/b1fb66b397734437b6239661839a7562 2024-11-07T12:25:40,084 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2365c66b71ff4e8c8f1d58274d0b3463, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/fa5d49bdd1954e26a6240499156f0614, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/622556e932c54d32841145b585b5d63b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/09a14c9cd6114398875d66d4dc9d1240, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5f34b5bb1754dfc82255ed6a438715b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/73fb949f482940b4ab746cceac8acc79, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ee849f44e56049279769a60c0f4eef8c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/facfff3a52244d9ea055220095a04212, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609] to archive 2024-11-07T12:25:40,085 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:25:40,086 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/7221ab34bf4b4e9488682c013a4caa73 2024-11-07T12:25:40,087 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/cc865f0e1f5c488a8d7ba45854d467c3 2024-11-07T12:25:40,088 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2365c66b71ff4e8c8f1d58274d0b3463 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2365c66b71ff4e8c8f1d58274d0b3463 2024-11-07T12:25:40,089 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/82779f30a1d04237a851cbe768386949 2024-11-07T12:25:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/a9beee83beff4fb78ca21bea145ced42 2024-11-07T12:25:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/fa5d49bdd1954e26a6240499156f0614 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/fa5d49bdd1954e26a6240499156f0614 2024-11-07T12:25:40,091 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/647958876659463fbc54abb2af06edcf 2024-11-07T12:25:40,092 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/9d26703e61f148b889f2500d99e18f24 2024-11-07T12:25:40,092 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/622556e932c54d32841145b585b5d63b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/622556e932c54d32841145b585b5d63b 2024-11-07T12:25:40,093 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/385576103b224d33848f0ebe04cce4ce 2024-11-07T12:25:40,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ae2d7991cd01443b91db25a5ff3828ca 2024-11-07T12:25:40,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/09a14c9cd6114398875d66d4dc9d1240 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/09a14c9cd6114398875d66d4dc9d1240 2024-11-07T12:25:40,095 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d0dc1b188c874572a523811ee5a046ec 2024-11-07T12:25:40,096 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2c29738f993c443c92543d4f4d79e753 2024-11-07T12:25:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5f34b5bb1754dfc82255ed6a438715b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5f34b5bb1754dfc82255ed6a438715b 2024-11-07T12:25:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/f5e8cb438b9f4ea5a093e7bdd51b1a7f 2024-11-07T12:25:40,098 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/3b169e52966b448ca710dc31d55e7c33 2024-11-07T12:25:40,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/dda494e1c66546aaa2bf988cac29cc5f 2024-11-07T12:25:40,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/73fb949f482940b4ab746cceac8acc79 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/73fb949f482940b4ab746cceac8acc79 2024-11-07T12:25:40,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/6a1479c6772344cab02b628df5a30d46 2024-11-07T12:25:40,101 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2a0885fa40364fabb764768420683508 2024-11-07T12:25:40,102 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ee849f44e56049279769a60c0f4eef8c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ee849f44e56049279769a60c0f4eef8c 2024-11-07T12:25:40,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/b5a39bb5d89a43ac9350510e8aeeccb2 2024-11-07T12:25:40,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/867755e76e7b4f569f3cf3304cb639c7 2024-11-07T12:25:40,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/facfff3a52244d9ea055220095a04212 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/facfff3a52244d9ea055220095a04212 2024-11-07T12:25:40,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/1da538cc68764e06a0cd84b02ba78b80 2024-11-07T12:25:40,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e35d606d1c03413aadff287f33bf4f87 2024-11-07T12:25:40,106 DEBUG [StoreCloser-TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/ef85026fe1ce461d8c7edda3062d8609 2024-11-07T12:25:40,108 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=1 2024-11-07T12:25:40,109 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. 2024-11-07T12:25:40,109 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for ea4184a46d62bfe1df974384dfe56d56: Waiting for close lock at 1730982338752Running coprocessor pre-close hooks at 1730982338752Disabling compacts and flushes for region at 1730982338752Disabling writes for close at 1730982338752Obtaining lock to block concurrent updates at 1730982338753 (+1 ms)Preparing flush snapshotting stores in ea4184a46d62bfe1df974384dfe56d56 at 1730982338753Finished memstore snapshotting TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730982338753Flushing stores of TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56. at 1730982338754 (+1 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/A: creating writer at 1730982338754Flushing ea4184a46d62bfe1df974384dfe56d56/A: appending metadata at 1730982338757 (+3 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/A: closing flushed file at 1730982338757Flushing ea4184a46d62bfe1df974384dfe56d56/B: creating writer at 1730982339168 (+411 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/B: appending metadata at 1730982339171 (+3 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/B: closing flushed file at 1730982339171Flushing ea4184a46d62bfe1df974384dfe56d56/C: creating writer at 1730982339579 (+408 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/C: appending metadata at 1730982339580 (+1 ms)Flushing ea4184a46d62bfe1df974384dfe56d56/C: closing flushed file at 1730982339580Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50df193: reopening flushed file at 1730982339988 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dbed42c: reopening flushed file at 1730982339993 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d5caf13: reopening flushed file at 1730982339998 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ea4184a46d62bfe1df974384dfe56d56 in 1250ms, sequenceid=414, compaction requested=true at 1730982340002 (+4 ms)Writing region close event to WAL at 1730982340107 (+105 ms)Running coprocessor post-close hooks at 1730982340109 (+2 ms)Closed at 1730982340109 2024-11-07T12:25:40,110 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:40,111 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=ea4184a46d62bfe1df974384dfe56d56, regionState=CLOSED 2024-11-07T12:25:40,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T12:25:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure ea4184a46d62bfe1df974384dfe56d56, server=8944cbb06cd2,46221,1730982182218 in 1.5130 sec 2024-11-07T12:25:40,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-07T12:25:40,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ea4184a46d62bfe1df974384dfe56d56, UNASSIGN in 1.5170 sec 2024-11-07T12:25:40,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T12:25:40,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-07T12:25:40,122 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982340121"}]},"ts":"1730982340121"} 2024-11-07T12:25:40,123 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:25:40,123 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:25:40,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-07T12:25:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T12:25:40,741 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:40,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:25:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:25:40,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,750 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:40,753 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/recovered.edits] 2024-11-07T12:25:40,758 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a552b15055c4d369b3d314e5d209348 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/8a552b15055c4d369b3d314e5d209348 2024-11-07T12:25:40,759 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba8f9f23219a4f108c577ceef78529a9 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ba8f9f23219a4f108c577ceef78529a9 2024-11-07T12:25:40,761 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ceab24a038af45178f67a38f04b48e04 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/A/ceab24a038af45178f67a38f04b48e04 2024-11-07T12:25:40,764 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/228b40ad12424942bd85b0a8c36dbb1d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/228b40ad12424942bd85b0a8c36dbb1d 2024-11-07T12:25:40,765 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/a78f9529e8ad42e8b37460b70cc0bd62 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/a78f9529e8ad42e8b37460b70cc0bd62 2024-11-07T12:25:40,767 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f27d288b576b4bc2a325d100810a30b2 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/B/f27d288b576b4bc2a325d100810a30b2 2024-11-07T12:25:40,769 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2bb240e3897e4d979cb8ea96983db7ad to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/2bb240e3897e4d979cb8ea96983db7ad 2024-11-07T12:25:40,771 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d03d08f6f0494ef6b1f86a00c98a77b7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/d03d08f6f0494ef6b1f86a00c98a77b7 2024-11-07T12:25:40,773 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e34591fff50a4249acf78bc0a012f8ae to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/C/e34591fff50a4249acf78bc0a012f8ae 2024-11-07T12:25:40,777 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/recovered.edits/417.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56/recovered.edits/417.seqid 2024-11-07T12:25:40,778 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/ea4184a46d62bfe1df974384dfe56d56 2024-11-07T12:25:40,778 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:25:40,780 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,783 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:25:40,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:25:40,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:25:40,786 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982340786"}]},"ts":"9223372036854775807"} 2024-11-07T12:25:40,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:25:40,788 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ea4184a46d62bfe1df974384dfe56d56, NAME => 'TestAcidGuarantees,,1730982312770.ea4184a46d62bfe1df974384dfe56d56.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:25:40,788 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:25:40,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982340788"}]},"ts":"9223372036854775807"} 2024-11-07T12:25:40,789 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:25:40,790 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-07T12:25:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T12:25:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:25:40,854 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:40,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 239), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=482 (was 561), ProcessCount=11 (was 11), AvailableMemoryMB=1514 (was 1602) 2024-11-07T12:25:40,874 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=482, ProcessCount=11, AvailableMemoryMB=1513 2024-11-07T12:25:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:25:40,875 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:25:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:40,876 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T12:25:40,877 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:40,877 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-07T12:25:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:40,877 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T12:25:40,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742373_1549 (size=959) 2024-11-07T12:25:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:41,287 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8 2024-11-07T12:25:41,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742374_1550 (size=53) 2024-11-07T12:25:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:41,722 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:25:41,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fb027f18a5ca907ae6c3eafe4c3b75bd, disabling compactions & flushes 2024-11-07T12:25:41,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:41,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:41,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. after waiting 0 ms 2024-11-07T12:25:41,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:41,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:41,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb027f18a5ca907ae6c3eafe4c3b75bd: Waiting for close lock at 1730982341722Disabling compacts and flushes for region at 1730982341722Disabling writes for close at 1730982341723 (+1 ms)Writing region close event to WAL at 1730982341723Closed at 1730982341723 2024-11-07T12:25:41,726 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T12:25:41,727 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730982341726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730982341726"}]},"ts":"1730982341726"} 2024-11-07T12:25:41,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T12:25:41,731 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T12:25:41,731 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982341731"}]},"ts":"1730982341731"} 2024-11-07T12:25:41,733 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T12:25:41,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, ASSIGN}] 2024-11-07T12:25:41,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, ASSIGN 2024-11-07T12:25:41,736 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, ASSIGN; state=OFFLINE, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=false 2024-11-07T12:25:41,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:25:41,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:41,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, ASSIGN because future has completed 2024-11-07T12:25:41,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:42,051 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:42,052 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:25:42,052 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,052 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:25:42,052 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,053 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,054 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,056 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName A 2024-11-07T12:25:42,056 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:42,056 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:42,057 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,058 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName B 2024-11-07T12:25:42,058 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:42,059 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:42,059 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,060 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName C 2024-11-07T12:25:42,060 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:42,060 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:42,061 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,061 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,062 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,063 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,063 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,063 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:25:42,064 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,066 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T12:25:42,066 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened fb027f18a5ca907ae6c3eafe4c3b75bd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71716379, jitterRate=0.06865732371807098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:25:42,066 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:42,067 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for fb027f18a5ca907ae6c3eafe4c3b75bd: Running coprocessor pre-open hook at 1730982342053Writing region info on filesystem at 1730982342053Initializing all the Stores at 1730982342053Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982342054 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982342054Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982342054Cleaning up temporary data from old regions at 1730982342063 (+9 ms)Running coprocessor post-open hooks at 1730982342066 (+3 ms)Region opened successfully at 1730982342067 (+1 ms) 2024-11-07T12:25:42,068 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., pid=144, masterSystemTime=1730982342045 2024-11-07T12:25:42,069 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:42,069 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:42,070 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=OPEN, openSeqNum=2, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:42,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:42,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T12:25:42,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 in 180 msec 2024-11-07T12:25:42,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-07T12:25:42,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, ASSIGN in 340 msec 2024-11-07T12:25:42,075 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T12:25:42,075 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982342075"}]},"ts":"1730982342075"} 2024-11-07T12:25:42,076 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T12:25:42,077 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T12:25:42,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2020 sec 2024-11-07T12:25:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T12:25:43,021 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:43,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6f989f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347601ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d47bf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,026 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:43,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:43,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:43,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T12:25:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8944cbb06cd2,37285,1730982181418 2024-11-07T12:25:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3392a1e1 2024-11-07T12:25:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T12:25:43,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T12:25:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T12:25:43,032 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T12:25:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T12:25:43,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742375_1551 (size=995) 2024-11-07T12:25:43,446 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T12:25:43,446 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T12:25:43,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:25:43,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE}] 2024-11-07T12:25:43,455 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE 2024-11-07T12:25:43,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:43,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE because future has completed 2024-11-07T12:25:43,458 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:25:43,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:43,610 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,611 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:25:43,611 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing fb027f18a5ca907ae6c3eafe4c3b75bd, disabling compactions & flushes 2024-11-07T12:25:43,611 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,611 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,611 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. after waiting 0 ms 2024-11-07T12:25:43,611 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,618 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T12:25:43,619 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,619 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for fb027f18a5ca907ae6c3eafe4c3b75bd: Waiting for close lock at 1730982343611Running coprocessor pre-close hooks at 1730982343611Disabling compacts and flushes for region at 1730982343611Disabling writes for close at 1730982343611Writing region close event to WAL at 1730982343612 (+1 ms)Running coprocessor post-close hooks at 1730982343619 (+7 ms)Closed at 1730982343619 2024-11-07T12:25:43,619 WARN [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: fb027f18a5ca907ae6c3eafe4c3b75bd to self. 2024-11-07T12:25:43,622 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,623 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=CLOSED 2024-11-07T12:25:43,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:43,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T12:25:43,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 in 169 msec 2024-11-07T12:25:43,631 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE; state=CLOSED, location=8944cbb06cd2,46221,1730982182218; forceNewPlan=false, retain=true 2024-11-07T12:25:43,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=OPENING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:43,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE because future has completed 2024-11-07T12:25:43,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:25:43,945 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,945 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} 2024-11-07T12:25:43,946 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,946 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T12:25:43,946 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,946 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,950 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,951 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName A 2024-11-07T12:25:43,952 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:43,953 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:43,953 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,954 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName B 2024-11-07T12:25:43,954 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:43,954 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:43,955 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,955 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb027f18a5ca907ae6c3eafe4c3b75bd columnFamilyName C 2024-11-07T12:25:43,955 DEBUG [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:43,956 INFO [StoreOpener-fb027f18a5ca907ae6c3eafe4c3b75bd-1 {}] regionserver.HStore(327): Store=fb027f18a5ca907ae6c3eafe4c3b75bd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T12:25:43,956 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,956 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,957 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,958 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,958 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,958 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T12:25:43,959 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,959 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened fb027f18a5ca907ae6c3eafe4c3b75bd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65717512, jitterRate=-0.020732760429382324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T12:25:43,959 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:43,960 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for fb027f18a5ca907ae6c3eafe4c3b75bd: Running coprocessor pre-open hook at 1730982343947Writing region info on filesystem at 1730982343947Initializing all the Stores at 1730982343948 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982343948Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982343949 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730982343949Cleaning up temporary data from old regions at 1730982343958 (+9 ms)Running coprocessor post-open hooks at 1730982343959 (+1 ms)Region opened successfully at 1730982343960 (+1 ms) 2024-11-07T12:25:43,961 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., pid=149, masterSystemTime=1730982343939 2024-11-07T12:25:43,962 DEBUG [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,962 INFO [RS_OPEN_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:43,963 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=OPEN, openSeqNum=5, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:43,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:25:43,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-07T12:25:43,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 in 179 msec 2024-11-07T12:25:43,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-07T12:25:43,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, REOPEN/MOVE in 512 msec 2024-11-07T12:25:43,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T12:25:43,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-07T12:25:43,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-07T12:25:43,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T12:25:43,972 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:43,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a209286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f4e194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c40987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa0b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb784ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a810d83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1130c997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,978 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275b4171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e6bcc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29acc1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,980 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288a1702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13899bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b691ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d5460a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f32d869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b0a686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409f646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593d5fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d05b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,989 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a03ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,991 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ccf2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27def64b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,993 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689dc940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448d7526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c440df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af60589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:43,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee9bb32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767c288b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:43,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8944cbb06cd2,37285,-1 for getting cluster id 2024-11-07T12:25:43,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T12:25:43,999 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59d2ef22-9ddc-4e31-b944-862110ef10b2' 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59d2ef22-9ddc-4e31-b944-862110ef10b2" 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68244f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8944cbb06cd2,37285,-1] 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T12:25:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:25:44,000 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T12:25:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebc62c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T12:25:44,001 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,001 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,001 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,002 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,002 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,002 DEBUG [Thread-2470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,003 DEBUG [Thread-2472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,004 DEBUG [Thread-2459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,004 DEBUG [Thread-2461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,006 DEBUG [Thread-2468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T12:25:44,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8944cbb06cd2,46221,1730982182218, seqNum=-1] 2024-11-07T12:25:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T12:25:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-07T12:25:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T12:25:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:44,010 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:44,011 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:44,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5] 2024-11-07T12:25:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:44,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T12:25:44,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982404037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982404040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982404041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a336a6c18f94ad69c90bfa73f6566ce_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982344008/Put/seqid=0 2024-11-07T12:25:44,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742376_1552 (size=12154) 2024-11-07T12:25:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982404150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982404150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982404150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982404150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982404150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982404360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982404360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982404360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982404361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982404361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:44,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a336a6c18f94ad69c90bfa73f6566ce_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a336a6c18f94ad69c90bfa73f6566ce_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:44,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/191626b859724f5eb1eb074e26e4ebf1, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:44,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/191626b859724f5eb1eb074e26e4ebf1 is 175, key is test_row_0/A:col10/1730982344008/Put/seqid=0 2024-11-07T12:25:44,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742377_1553 (size=30955) 2024-11-07T12:25:44,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/191626b859724f5eb1eb074e26e4ebf1 2024-11-07T12:25:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1058be1aa18448bc86e81bd7cd072b3d is 50, key is test_row_0/B:col10/1730982344008/Put/seqid=0 2024-11-07T12:25:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742378_1554 (size=12001) 2024-11-07T12:25:44,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982404671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:44,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1058be1aa18448bc86e81bd7cd072b3d 2024-11-07T12:25:44,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d06c65a93cb1437a86bc5172fd9657b7 is 50, key is test_row_0/C:col10/1730982344008/Put/seqid=0 2024-11-07T12:25:44,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742379_1555 (size=12001) 2024-11-07T12:25:44,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:44,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:45,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:45,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:45,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:45,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982405181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982405182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982405182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982405182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982405182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:45,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:45,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:45,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d06c65a93cb1437a86bc5172fd9657b7 2024-11-07T12:25:45,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/191626b859724f5eb1eb074e26e4ebf1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1 2024-11-07T12:25:45,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1, entries=150, sequenceid=17, filesize=30.2 K 2024-11-07T12:25:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1058be1aa18448bc86e81bd7cd072b3d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d 2024-11-07T12:25:45,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T12:25:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d06c65a93cb1437a86bc5172fd9657b7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7 2024-11-07T12:25:45,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T12:25:45,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1312ms, sequenceid=17, compaction requested=false 2024-11-07T12:25:45,328 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T12:25:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:45,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T12:25:45,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:45,384 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:25:45,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca22c4be549f4bbb8d73b6a645c8acef_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982344028/Put/seqid=0 2024-11-07T12:25:45,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742380_1556 (size=12154) 2024-11-07T12:25:45,626 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T12:25:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:45,798 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca22c4be549f4bbb8d73b6a645c8acef_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca22c4be549f4bbb8d73b6a645c8acef_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:45,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/ed34c46c7daf4700bf616af69ac193f8, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/ed34c46c7daf4700bf616af69ac193f8 is 175, key is test_row_0/A:col10/1730982344028/Put/seqid=0 2024-11-07T12:25:45,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742381_1557 (size=30955) 2024-11-07T12:25:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:46,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982406196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982406197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982406198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/ed34c46c7daf4700bf616af69ac193f8 2024-11-07T12:25:46,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/fd1a1da12eaf4cbfac758205ad9e3228 is 50, key is test_row_0/B:col10/1730982344028/Put/seqid=0 2024-11-07T12:25:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742382_1558 (size=12001) 2024-11-07T12:25:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982406310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982406310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982406311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982406311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982406311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982406520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982406520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982406521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982406521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982406521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,614 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/fd1a1da12eaf4cbfac758205ad9e3228 2024-11-07T12:25:46,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d939df2ddfd14ce5880bdea9a86a55d0 is 50, key is test_row_0/C:col10/1730982344028/Put/seqid=0 2024-11-07T12:25:46,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742383_1559 (size=12001) 2024-11-07T12:25:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982406830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982406830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982406830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982406830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982406830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d939df2ddfd14ce5880bdea9a86a55d0 2024-11-07T12:25:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/ed34c46c7daf4700bf616af69ac193f8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8 2024-11-07T12:25:47,031 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T12:25:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/fd1a1da12eaf4cbfac758205ad9e3228 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228 2024-11-07T12:25:47,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:25:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d939df2ddfd14ce5880bdea9a86a55d0 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0 2024-11-07T12:25:47,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T12:25:47,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1653ms, sequenceid=40, compaction requested=false 2024-11-07T12:25:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:47,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-07T12:25:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-07T12:25:47,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-07T12:25:47,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0280 sec 2024-11-07T12:25:47,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0350 sec 2024-11-07T12:25:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:47,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:47,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T12:25:47,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072906d65119eb479dbf94c8c656ddf3de_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742384_1560 (size=12154) 2024-11-07T12:25:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982407358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982407359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982407360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982407361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982407362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982407471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982407471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982407471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982407471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982407471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982407681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982407681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982407681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982407681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982407681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:47,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072906d65119eb479dbf94c8c656ddf3de_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072906d65119eb479dbf94c8c656ddf3de_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:47,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7817fffef8874ea38595035c6d041ecb, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:47,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7817fffef8874ea38595035c6d041ecb is 175, key is test_row_0/A:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742385_1561 (size=30955) 2024-11-07T12:25:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982407991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7817fffef8874ea38595035c6d041ecb 2024-11-07T12:25:48,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/a8d07257e3924dafa3f6ff4bd1ad403a is 50, key is test_row_0/B:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742386_1562 (size=12001) 2024-11-07T12:25:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T12:25:48,181 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:48,181 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-07T12:25:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:48,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:48,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:48,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:48,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T12:25:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T12:25:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982408501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982408501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982408501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982408501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982408501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:48,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/a8d07257e3924dafa3f6ff4bd1ad403a 2024-11-07T12:25:48,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/73d84e2e51a74e8d88e08f5fff0e51e8 is 50, key is test_row_0/C:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742387_1563 (size=12001) 2024-11-07T12:25:48,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/73d84e2e51a74e8d88e08f5fff0e51e8 2024-11-07T12:25:48,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7817fffef8874ea38595035c6d041ecb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb 2024-11-07T12:25:48,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb, entries=150, sequenceid=54, filesize=30.2 K 2024-11-07T12:25:48,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/a8d07257e3924dafa3f6ff4bd1ad403a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a 2024-11-07T12:25:48,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:25:48,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/73d84e2e51a74e8d88e08f5fff0e51e8 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8 2024-11-07T12:25:48,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T12:25:48,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1247ms, sequenceid=54, compaction requested=true 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:48,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:25:48,590 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:25:48,590 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,590 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,591 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=90.7 K 2024-11-07T12:25:48,591 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.2 K 2024-11-07T12:25:48,591 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb] 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1058be1aa18448bc86e81bd7cd072b3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730982344008 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191626b859724f5eb1eb074e26e4ebf1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730982344008 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1a1da12eaf4cbfac758205ad9e3228, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982344028 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed34c46c7daf4700bf616af69ac193f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982344028 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting a8d07257e3924dafa3f6ff4bd1ad403a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:48,591 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7817fffef8874ea38595035c6d041ecb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:48,595 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:48,596 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:48,596 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107814a872387424586a60a6458fe0c576f_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:48,596 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/fc9301bab4944148af965dce0bb80c73 is 50, key is test_row_0/B:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:48,598 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107814a872387424586a60a6458fe0c576f_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:48,598 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107814a872387424586a60a6458fe0c576f_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:48,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742388_1564 (size=12104) 2024-11-07T12:25:48,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742389_1565 (size=4469) 2024-11-07T12:25:48,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T12:25:48,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:48,640 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T12:25:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078d516c655d4d41fb8d4aaef341ac35c2_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982347361/Put/seqid=0 2024-11-07T12:25:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742390_1566 (size=12154) 2024-11-07T12:25:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:49,002 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/fc9301bab4944148af965dce0bb80c73 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fc9301bab4944148af965dce0bb80c73 2024-11-07T12:25:49,005 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into fc9301bab4944148af965dce0bb80c73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:49,006 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:49,006 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=13, startTime=1730982348590; duration=0sec 2024-11-07T12:25:49,006 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:49,006 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:25:49,006 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:49,006 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#475 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:49,006 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/fde27bb3124b4b42890e5fbc19cb0bdd is 175, key is test_row_0/A:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:49,007 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:49,007 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:25:49,007 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:49,007 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.2 K 2024-11-07T12:25:49,007 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d06c65a93cb1437a86bc5172fd9657b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730982344008 2024-11-07T12:25:49,008 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d939df2ddfd14ce5880bdea9a86a55d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730982344028 2024-11-07T12:25:49,008 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d84e2e51a74e8d88e08f5fff0e51e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:49,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742391_1567 (size=31058) 2024-11-07T12:25:49,012 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/fde27bb3124b4b42890e5fbc19cb0bdd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd 2024-11-07T12:25:49,016 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:49,016 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into fde27bb3124b4b42890e5fbc19cb0bdd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:49,016 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:49,016 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=13, startTime=1730982348590; duration=0sec 2024-11-07T12:25:49,016 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ffa0a08819e142d2a0a1fe041d6574a5 is 50, key is test_row_0/C:col10/1730982346198/Put/seqid=0 2024-11-07T12:25:49,016 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:49,016 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:25:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742392_1568 (size=12104) 2024-11-07T12:25:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:49,051 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078d516c655d4d41fb8d4aaef341ac35c2_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d516c655d4d41fb8d4aaef341ac35c2_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/46e0bc5dbd494a95adc461026a8fdfda, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:49,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/46e0bc5dbd494a95adc461026a8fdfda is 175, key is test_row_0/A:col10/1730982347361/Put/seqid=0 2024-11-07T12:25:49,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742393_1569 (size=30955) 2024-11-07T12:25:49,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/46e0bc5dbd494a95adc461026a8fdfda 2024-11-07T12:25:49,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/21edf8f4074e4440bb7a5788fa101af7 is 50, key is test_row_0/B:col10/1730982347361/Put/seqid=0 2024-11-07T12:25:49,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742394_1570 (size=12001) 2024-11-07T12:25:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:49,421 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ffa0a08819e142d2a0a1fe041d6574a5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ffa0a08819e142d2a0a1fe041d6574a5 2024-11-07T12:25:49,424 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into ffa0a08819e142d2a0a1fe041d6574a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:49,424 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:49,425 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=13, startTime=1730982348590; duration=0sec 2024-11-07T12:25:49,425 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:49,425 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:25:49,465 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/21edf8f4074e4440bb7a5788fa101af7 2024-11-07T12:25:49,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e3a03316122d4aa8824f08d8d080d068 is 50, key is test_row_0/C:col10/1730982347361/Put/seqid=0 2024-11-07T12:25:49,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742395_1571 (size=12001) 2024-11-07T12:25:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:49,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:49,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982409521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982409522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982409522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982409523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982409524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982409630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982409630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982409630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982409631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982409630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982409841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982409841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982409841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982409841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982409842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:49,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e3a03316122d4aa8824f08d8d080d068 2024-11-07T12:25:49,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/46e0bc5dbd494a95adc461026a8fdfda as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda 2024-11-07T12:25:49,879 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda, entries=150, sequenceid=76, filesize=30.2 K 2024-11-07T12:25:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/21edf8f4074e4440bb7a5788fa101af7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7 2024-11-07T12:25:49,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:25:49,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e3a03316122d4aa8824f08d8d080d068 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068 2024-11-07T12:25:49,886 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T12:25:49,887 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1247ms, sequenceid=76, compaction requested=false 2024-11-07T12:25:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:49,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-07T12:25:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-07T12:25:49,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T12:25:49,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-07T12:25:49,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7090 sec 2024-11-07T12:25:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:50,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:50,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:25:50,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a120578d63d4a95a807019413fff81a_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:50,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742396_1572 (size=14594) 2024-11-07T12:25:50,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982410173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982410173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982410173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982410173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982410174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982410280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982410281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T12:25:50,331 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:50,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-07T12:25:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:50,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:50,333 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:50,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:50,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982410491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982410491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982410491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982410491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982410491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:50,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a120578d63d4a95a807019413fff81a_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a120578d63d4a95a807019413fff81a_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:50,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/86772dd5b05d42f19401e8b668900e40, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:50,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/86772dd5b05d42f19401e8b668900e40 is 175, key is test_row_0/A:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:50,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742397_1573 (size=39549) 2024-11-07T12:25:50,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:50,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982410800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982410801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982410801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982410801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982410801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:50,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:50,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:50,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:50,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/86772dd5b05d42f19401e8b668900e40 2024-11-07T12:25:50,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/4beddaafc3df41c89bc20acfbb69eb59 is 50, key is test_row_0/B:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742398_1574 (size=12001) 2024-11-07T12:25:51,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982411311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982411311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982411311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982411311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982411312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:51,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/4beddaafc3df41c89bc20acfbb69eb59 2024-11-07T12:25:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:51,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/2f91068d812c4490b4f62c270e62f874 is 50, key is test_row_0/C:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742399_1575 (size=12001) 2024-11-07T12:25:51,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:51,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T12:25:51,814 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T12:25:51,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/2f91068d812c4490b4f62c270e62f874 2024-11-07T12:25:51,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/86772dd5b05d42f19401e8b668900e40 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40 2024-11-07T12:25:51,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40, entries=200, sequenceid=96, filesize=38.6 K 2024-11-07T12:25:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/4beddaafc3df41c89bc20acfbb69eb59 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59 2024-11-07T12:25:51,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59, entries=150, sequenceid=96, filesize=11.7 K 2024-11-07T12:25:51,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/2f91068d812c4490b4f62c270e62f874 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874 2024-11-07T12:25:51,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874, entries=150, sequenceid=96, filesize=11.7 K 2024-11-07T12:25:51,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1735ms, sequenceid=96, compaction requested=true 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:51,888 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:51,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:51,888 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:51,889 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:51,889 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:51,889 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:25:51,889 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:25:51,890 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,890 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,890 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=99.2 K 2024-11-07T12:25:51,890 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fc9301bab4944148af965dce0bb80c73, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.3 K 2024-11-07T12:25:51,890 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40] 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9301bab4944148af965dce0bb80c73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde27bb3124b4b42890e5fbc19cb0bdd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 21edf8f4074e4440bb7a5788fa101af7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730982347356 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e0bc5dbd494a95adc461026a8fdfda, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730982347356 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 4beddaafc3df41c89bc20acfbb69eb59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:51,890 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86772dd5b05d42f19401e8b668900e40, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:51,894 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:51,895 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107caad981b3c3d4e09a373654c0ecb71b9_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:51,896 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:51,896 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/512e9ad4096c467491ec048890e9c765 is 50, key is test_row_0/B:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:51,897 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107caad981b3c3d4e09a373654c0ecb71b9_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:51,897 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107caad981b3c3d4e09a373654c0ecb71b9_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:51,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742400_1576 (size=12207) 2024-11-07T12:25:51,902 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/512e9ad4096c467491ec048890e9c765 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/512e9ad4096c467491ec048890e9c765 2024-11-07T12:25:51,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742401_1577 (size=4469) 2024-11-07T12:25:51,905 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into 512e9ad4096c467491ec048890e9c765(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:51,906 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=13, startTime=1730982351888; duration=0sec 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:51,906 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:25:51,906 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,906 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ffa0a08819e142d2a0a1fe041d6574a5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.3 K 2024-11-07T12:25:51,907 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ffa0a08819e142d2a0a1fe041d6574a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730982346193 2024-11-07T12:25:51,907 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a03316122d4aa8824f08d8d080d068, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730982347356 2024-11-07T12:25:51,907 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f91068d812c4490b4f62c270e62f874, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:51,911 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#485 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:51,912 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/23c58f3464274ade8b6f1d4d0b52be07 is 50, key is test_row_0/C:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742402_1578 (size=12207) 2024-11-07T12:25:51,925 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/23c58f3464274ade8b6f1d4d0b52be07 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/23c58f3464274ade8b6f1d4d0b52be07 2024-11-07T12:25:51,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T12:25:51,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:51,926 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T12:25:51,930 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into 23c58f3464274ade8b6f1d4d0b52be07(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:51,930 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:51,930 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=13, startTime=1730982351888; duration=0sec 2024-11-07T12:25:51,931 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:51,931 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:25:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107346ec64b9bd741789fa5efe448224b7a_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982350172/Put/seqid=0 2024-11-07T12:25:51,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742403_1579 (size=12154) 2024-11-07T12:25:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:51,940 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107346ec64b9bd741789fa5efe448224b7a_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107346ec64b9bd741789fa5efe448224b7a_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2546b5461d8b4a909aa3d86410c863cd, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2546b5461d8b4a909aa3d86410c863cd is 175, key is test_row_0/A:col10/1730982350172/Put/seqid=0 2024-11-07T12:25:51,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742404_1580 (size=30955) 2024-11-07T12:25:52,305 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:52,306 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/fb52634bb22e4f1d9194e9a6b062f5cc is 175, key is test_row_0/A:col10/1730982350153/Put/seqid=0 2024-11-07T12:25:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742405_1581 (size=31161) 2024-11-07T12:25:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:52,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982412335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982412338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982412339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982412340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,345 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2546b5461d8b4a909aa3d86410c863cd 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982412341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e1e8e58f9bef4f28bb499501ba91520c is 50, key is test_row_0/B:col10/1730982350172/Put/seqid=0 2024-11-07T12:25:52,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742406_1582 (size=12001) 2024-11-07T12:25:52,353 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e1e8e58f9bef4f28bb499501ba91520c 2024-11-07T12:25:52,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/3dd7acbd291e41c28e87460eb4a49487 is 50, key is test_row_0/C:col10/1730982350172/Put/seqid=0 2024-11-07T12:25:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742407_1583 (size=12001) 2024-11-07T12:25:52,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982412451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982412661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982412661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982412661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982412661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982412661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,712 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/fb52634bb22e4f1d9194e9a6b062f5cc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc 2024-11-07T12:25:52,715 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into fb52634bb22e4f1d9194e9a6b062f5cc(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:52,715 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:52,715 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=13, startTime=1730982351888; duration=0sec 2024-11-07T12:25:52,715 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:52,715 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:25:52,761 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/3dd7acbd291e41c28e87460eb4a49487 2024-11-07T12:25:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2546b5461d8b4a909aa3d86410c863cd as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd 2024-11-07T12:25:52,767 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd, entries=150, sequenceid=117, filesize=30.2 K 2024-11-07T12:25:52,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e1e8e58f9bef4f28bb499501ba91520c as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c 2024-11-07T12:25:52,770 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T12:25:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/3dd7acbd291e41c28e87460eb4a49487 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487 2024-11-07T12:25:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T12:25:52,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for fb027f18a5ca907ae6c3eafe4c3b75bd in 848ms, sequenceid=117, compaction requested=false 2024-11-07T12:25:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-07T12:25:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-07T12:25:52,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-07T12:25:52,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4420 sec 2024-11-07T12:25:52,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.4460 sec 2024-11-07T12:25:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:52,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:52,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T12:25:52,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071ad2f7c544df453c9926a9a17b8d40e3_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982412984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982412984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982412987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982412987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982412988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742408_1584 (size=12304) 2024-11-07T12:25:53,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982413091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982413091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982413100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982413100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982413100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982413301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982413301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982413310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982413310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982413310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:53,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071ad2f7c544df453c9926a9a17b8d40e3_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071ad2f7c544df453c9926a9a17b8d40e3_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:53,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/d9e34322447642d8a7783982922fa890, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:53,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/d9e34322447642d8a7783982922fa890 is 175, key is test_row_0/A:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:53,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742409_1585 (size=31105) 2024-11-07T12:25:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982413610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982413611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982413621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982413621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982413621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:53,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/d9e34322447642d8a7783982922fa890 2024-11-07T12:25:53,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1320d2ac4c744b2db8f7e9c2f1424908 is 50, key is test_row_0/B:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:53,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742410_1586 (size=12151) 2024-11-07T12:25:54,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982414121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982414121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982414131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:54,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1320d2ac4c744b2db8f7e9c2f1424908 2024-11-07T12:25:54,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/24d18255d8724d3eb1562514c681aa2d is 50, key is test_row_0/C:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:54,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742411_1587 (size=12151) 2024-11-07T12:25:54,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/24d18255d8724d3eb1562514c681aa2d 2024-11-07T12:25:54,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/d9e34322447642d8a7783982922fa890 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890 2024-11-07T12:25:54,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890, entries=150, sequenceid=137, filesize=30.4 K 2024-11-07T12:25:54,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/1320d2ac4c744b2db8f7e9c2f1424908 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908 2024-11-07T12:25:54,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908, entries=150, sequenceid=137, filesize=11.9 K 2024-11-07T12:25:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/24d18255d8724d3eb1562514c681aa2d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d 2024-11-07T12:25:54,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d, entries=150, sequenceid=137, filesize=11.9 K 2024-11-07T12:25:54,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1263ms, sequenceid=137, compaction requested=true 2024-11-07T12:25:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:25:54,237 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:54,237 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:54,237 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/512e9ad4096c467491ec048890e9c765, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.5 K 2024-11-07T12:25:54,237 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=91.0 K 2024-11-07T12:25:54,237 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:54,237 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890] 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 512e9ad4096c467491ec048890e9c765, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb52634bb22e4f1d9194e9a6b062f5cc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2546b5461d8b4a909aa3d86410c863cd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730982350166 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e8e58f9bef4f28bb499501ba91520c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730982350166 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e34322447642d8a7783982922fa890, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:25:54,238 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 1320d2ac4c744b2db8f7e9c2f1424908, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:25:54,242 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:54,244 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:54,244 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bced3817b1ee4d93bf03279f26ecb5e9_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:54,244 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/78cbd25a9f264afe8f78374b346b608a is 50, key is test_row_0/B:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:54,246 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bced3817b1ee4d93bf03279f26ecb5e9_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:54,246 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bced3817b1ee4d93bf03279f26ecb5e9_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:54,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742412_1588 (size=12459) 2024-11-07T12:25:54,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742413_1589 (size=4469) 2024-11-07T12:25:54,265 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#492 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:54,266 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc is 175, key is test_row_0/A:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:54,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742414_1590 (size=31413) 2024-11-07T12:25:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T12:25:54,501 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:54,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-07T12:25:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:54,503 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:54,503 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:54,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:54,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T12:25:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:54,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:25:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110793d7168433a84db48f56e3c43c9a6f37_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982352982/Put/seqid=0 2024-11-07T12:25:54,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742415_1591 (size=12304) 2024-11-07T12:25:54,668 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/78cbd25a9f264afe8f78374b346b608a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/78cbd25a9f264afe8f78374b346b608a 2024-11-07T12:25:54,671 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into 78cbd25a9f264afe8f78374b346b608a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:54,672 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:54,672 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=13, startTime=1730982354237; duration=0sec 2024-11-07T12:25:54,672 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:54,672 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:25:54,672 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:25:54,672 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc 2024-11-07T12:25:54,673 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:25:54,673 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:25:54,673 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:54,673 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/23c58f3464274ade8b6f1d4d0b52be07, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=35.5 K 2024-11-07T12:25:54,673 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c58f3464274ade8b6f1d4d0b52be07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1730982349523 2024-11-07T12:25:54,673 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dd7acbd291e41c28e87460eb4a49487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730982350166 2024-11-07T12:25:54,674 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d18255d8724d3eb1562514c681aa2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:25:54,676 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into 2fd3098aacf347b9a2ba2dd8cbf8dbfc(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:54,676 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:54,676 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=13, startTime=1730982354236; duration=0sec 2024-11-07T12:25:54,676 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:54,676 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:25:54,679 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:54,679 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/8f606cfc04524491a6a46a93c4bc04cc is 50, key is test_row_0/C:col10/1730982352972/Put/seqid=0 2024-11-07T12:25:54,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742416_1592 (size=12459) 2024-11-07T12:25:54,705 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/8f606cfc04524491a6a46a93c4bc04cc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8f606cfc04524491a6a46a93c4bc04cc 2024-11-07T12:25:54,708 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into 8f606cfc04524491a6a46a93c4bc04cc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:25:54,708 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:54,708 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=13, startTime=1730982354237; duration=0sec 2024-11-07T12:25:54,708 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:54,708 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:25:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:55,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:55,066 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110793d7168433a84db48f56e3c43c9a6f37_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110793d7168433a84db48f56e3c43c9a6f37_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81d1f36b6b6e4eea8c4d9cc033639783, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81d1f36b6b6e4eea8c4d9cc033639783 is 175, key is test_row_0/A:col10/1730982352982/Put/seqid=0 2024-11-07T12:25:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742417_1593 (size=31105) 2024-11-07T12:25:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:55,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982415149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982415150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982415150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982415150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,153 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:55,153 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982415152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982415261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982415261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982415261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81d1f36b6b6e4eea8c4d9cc033639783 2024-11-07T12:25:55,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982415471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982415471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982415471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/04cbc2bf920d48c2ae8d4cc739c953e4 is 50, key is test_row_0/B:col10/1730982352982/Put/seqid=0 2024-11-07T12:25:55,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742418_1594 (size=12151) 2024-11-07T12:25:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982415781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982415781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982415781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:55,880 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/04cbc2bf920d48c2ae8d4cc739c953e4 2024-11-07T12:25:55,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ae3640f9b50c4cabb69a4771657ac921 is 50, key is test_row_0/C:col10/1730982352982/Put/seqid=0 2024-11-07T12:25:55,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742419_1595 (size=12151) 2024-11-07T12:25:56,289 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ae3640f9b50c4cabb69a4771657ac921 2024-11-07T12:25:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81d1f36b6b6e4eea8c4d9cc033639783 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783 2024-11-07T12:25:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982416290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982416291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982416291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:56,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783, entries=150, sequenceid=155, filesize=30.4 K 2024-11-07T12:25:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/04cbc2bf920d48c2ae8d4cc739c953e4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4 2024-11-07T12:25:56,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:25:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ae3640f9b50c4cabb69a4771657ac921 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921 2024-11-07T12:25:56,301 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T12:25:56,301 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1646ms, sequenceid=155, compaction requested=false 2024-11-07T12:25:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-07T12:25:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-07T12:25:56,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T12:25:56,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-07T12:25:56,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8040 sec 2024-11-07T12:25:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T12:25:56,650 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:56,651 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-07T12:25:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:56,652 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:56,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:56,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:56,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T12:25:56,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:56,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T12:25:56,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743b4eb941efe4694b5992bd1fc73250f_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982355151/Put/seqid=0 2024-11-07T12:25:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742420_1596 (size=12304) 2024-11-07T12:25:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:57,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982417195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982417198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:57,222 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743b4eb941efe4694b5992bd1fc73250f_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743b4eb941efe4694b5992bd1fc73250f_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/b4c3a0670ab149d6995b78065285fcff, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/b4c3a0670ab149d6995b78065285fcff is 175, key is test_row_0/A:col10/1730982355151/Put/seqid=0 2024-11-07T12:25:57,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742421_1597 (size=31105) 2024-11-07T12:25:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:57,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982417300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982417301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,303 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:57,303 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982417311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,313 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982417311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982417311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982417521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982417521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/b4c3a0670ab149d6995b78065285fcff 2024-11-07T12:25:57,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e600814dd4ce4df5824045ea74b35b6e is 50, key is test_row_0/B:col10/1730982355151/Put/seqid=0 2024-11-07T12:25:57,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742422_1598 (size=12151) 2024-11-07T12:25:57,636 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e600814dd4ce4df5824045ea74b35b6e 2024-11-07T12:25:57,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/be29a315085a4875a31335bb105424d7 is 50, key is test_row_0/C:col10/1730982355151/Put/seqid=0 2024-11-07T12:25:57,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742423_1599 (size=12151) 2024-11-07T12:25:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:57,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982417831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982417831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,044 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/be29a315085a4875a31335bb105424d7 2024-11-07T12:25:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/b4c3a0670ab149d6995b78065285fcff as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff 2024-11-07T12:25:58,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff, entries=150, sequenceid=176, filesize=30.4 K 2024-11-07T12:25:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/e600814dd4ce4df5824045ea74b35b6e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e 2024-11-07T12:25:58,053 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e, entries=150, sequenceid=176, filesize=11.9 K 2024-11-07T12:25:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/be29a315085a4875a31335bb105424d7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7 2024-11-07T12:25:58,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-07T12:25:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1251ms, sequenceid=176, compaction requested=true 2024-11-07T12:25:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-07T12:25:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-07T12:25:58,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-07T12:25:58,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-07T12:25:58,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4080 sec 2024-11-07T12:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:25:58,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T12:25:58,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072f4f5a7df4964fe28dc574312e679875_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982358343/Put/seqid=0 2024-11-07T12:25:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742424_1600 (size=14794) 2024-11-07T12:25:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982418375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982418374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982418481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982418481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982418691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982418691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:58,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:25:58,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072f4f5a7df4964fe28dc574312e679875_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072f4f5a7df4964fe28dc574312e679875_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:25:58,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/a488f0778169490bbd71e55124667060, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:25:58,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/a488f0778169490bbd71e55124667060 is 175, key is test_row_0/A:col10/1730982358343/Put/seqid=0 2024-11-07T12:25:58,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742425_1601 (size=39749) 2024-11-07T12:25:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T12:25:58,801 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:25:58,801 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:25:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T12:25:58,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:25:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:25:58,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:25:58,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:25:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:25:58,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:58,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982419000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982419000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:25:59,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/a488f0778169490bbd71e55124667060 2024-11-07T12:25:59,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/cb16919fd02d4ba287a413c2aacbf5be is 50, key is test_row_0/B:col10/1730982358343/Put/seqid=0 2024-11-07T12:25:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742426_1602 (size=12151) 2024-11-07T12:25:59,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982419321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982419321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,323 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:59,323 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982419340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,341 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:25:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:25:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982419511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982419511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:25:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:25:59,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/cb16919fd02d4ba287a413c2aacbf5be 2024-11-07T12:25:59,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e41401e5e60648ea8289414c826d0d8d is 50, key is test_row_0/C:col10/1730982358343/Put/seqid=0 2024-11-07T12:25:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742427_1603 (size=12151) 2024-11-07T12:25:59,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:25:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:25:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:25:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:25:59,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e41401e5e60648ea8289414c826d0d8d 2024-11-07T12:25:59,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/a488f0778169490bbd71e55124667060 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060 2024-11-07T12:25:59,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060, entries=200, sequenceid=196, filesize=38.8 K 2024-11-07T12:25:59,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/cb16919fd02d4ba287a413c2aacbf5be as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be 2024-11-07T12:25:59,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:25:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/e41401e5e60648ea8289414c826d0d8d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d 2024-11-07T12:25:59,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T12:25:59,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1645ms, sequenceid=196, compaction requested=true 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:25:59,988 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:25:59,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:25:59,988 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:25:59,989 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:25:59,989 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:25:59,989 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:25:59,989 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:25:59,989 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,989 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,989 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/78cbd25a9f264afe8f78374b346b608a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=47.8 K 2024-11-07T12:25:59,989 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=130.2 K 2024-11-07T12:25:59,989 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:25:59,989 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060] 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cbd25a9f264afe8f78374b346b608a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fd3098aacf347b9a2ba2dd8cbf8dbfc, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d1f36b6b6e4eea8c4d9cc033639783, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982352982 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 04cbc2bf920d48c2ae8d4cc739c953e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982352982 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c3a0670ab149d6995b78065285fcff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730982355148 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e600814dd4ce4df5824045ea74b35b6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730982355148 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting a488f0778169490bbd71e55124667060, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:25:59,990 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting cb16919fd02d4ba287a413c2aacbf5be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:25:59,997 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:25:59,997 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/6ee288e25c5e433cbc284fff536bf462 is 50, key is test_row_0/B:col10/1730982358343/Put/seqid=0 2024-11-07T12:25:59,998 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:00,000 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411071e0fd656e0db4b50bfb04927492cac9a_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:00,002 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411071e0fd656e0db4b50bfb04927492cac9a_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:00,002 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071e0fd656e0db4b50bfb04927492cac9a_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:00,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742428_1604 (size=12595) 2024-11-07T12:26:00,010 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/6ee288e25c5e433cbc284fff536bf462 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6ee288e25c5e433cbc284fff536bf462 2024-11-07T12:26:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742429_1605 (size=4469) 2024-11-07T12:26:00,015 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#505 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:00,015 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into 6ee288e25c5e433cbc284fff536bf462(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:00,015 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8e0a80f0ad4243a8b451446fb74df3c6 is 175, key is test_row_0/A:col10/1730982358343/Put/seqid=0 2024-11-07T12:26:00,015 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:00,015 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=12, startTime=1730982359988; duration=0sec 2024-11-07T12:26:00,015 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:26:00,015 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:26:00,015 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T12:26:00,016 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T12:26:00,016 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:26:00,017 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:00,017 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8f606cfc04524491a6a46a93c4bc04cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=47.8 K 2024-11-07T12:26:00,017 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f606cfc04524491a6a46a93c4bc04cc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1730982352340 2024-11-07T12:26:00,017 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3640f9b50c4cabb69a4771657ac921, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730982352982 2024-11-07T12:26:00,017 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting be29a315085a4875a31335bb105424d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730982355148 2024-11-07T12:26:00,018 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting e41401e5e60648ea8289414c826d0d8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:26:00,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742430_1606 (size=31549) 2024-11-07T12:26:00,022 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8e0a80f0ad4243a8b451446fb74df3c6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6 2024-11-07T12:26:00,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T12:26:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:00,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T12:26:00,027 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:00,028 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ec2605f9d3ab49f2b4b3f78028e8aa06 is 50, key is test_row_0/C:col10/1730982358343/Put/seqid=0 2024-11-07T12:26:00,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b0bae90bdf43427c81affa1da6119e45_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982358374/Put/seqid=0 2024-11-07T12:26:00,029 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into 8e0a80f0ad4243a8b451446fb74df3c6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:00,029 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:00,029 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=12, startTime=1730982359988; duration=0sec 2024-11-07T12:26:00,029 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:00,029 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:26:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742431_1607 (size=12595) 2024-11-07T12:26:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742432_1608 (size=12304) 2024-11-07T12:26:00,034 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ec2605f9d3ab49f2b4b3f78028e8aa06 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ec2605f9d3ab49f2b4b3f78028e8aa06 2024-11-07T12:26:00,037 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into ec2605f9d3ab49f2b4b3f78028e8aa06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:00,037 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:00,038 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=12, startTime=1730982359988; duration=0sec 2024-11-07T12:26:00,038 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:00,038 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:26:00,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:26:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b0bae90bdf43427c81affa1da6119e45_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0bae90bdf43427c81affa1da6119e45_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:00,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/dd1372277ed5407182d3f196e3943930, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/dd1372277ed5407182d3f196e3943930 is 175, key is test_row_0/A:col10/1730982358374/Put/seqid=0 2024-11-07T12:26:00,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742433_1609 (size=31105) 2024-11-07T12:26:00,440 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/dd1372277ed5407182d3f196e3943930 2024-11-07T12:26:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/6f34121aa3f7451db349e4ea4bf39ef5 is 50, key is test_row_0/B:col10/1730982358374/Put/seqid=0 2024-11-07T12:26:00,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742434_1610 (size=12151) 2024-11-07T12:26:00,492 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T12:26:00,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:26:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982420563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982420564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982420670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982420670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,850 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/6f34121aa3f7451db349e4ea4bf39ef5 2024-11-07T12:26:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d4736b5f447b45dc9eda8ae730230d13 is 50, key is test_row_0/C:col10/1730982358374/Put/seqid=0 2024-11-07T12:26:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742435_1611 (size=12151) 2024-11-07T12:26:00,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982420881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982420881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:26:01,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982421191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982421191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:01,258 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d4736b5f447b45dc9eda8ae730230d13 2024-11-07T12:26:01,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/dd1372277ed5407182d3f196e3943930 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930 2024-11-07T12:26:01,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930, entries=150, sequenceid=213, filesize=30.4 K 2024-11-07T12:26:01,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/6f34121aa3f7451db349e4ea4bf39ef5 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5 2024-11-07T12:26:01,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T12:26:01,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d4736b5f447b45dc9eda8ae730230d13 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13 2024-11-07T12:26:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T12:26:01,270 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1247ms, sequenceid=213, compaction requested=false 2024-11-07T12:26:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T12:26:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T12:26:01,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T12:26:01,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4690 sec 2024-11-07T12:26:01,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.4720 sec 2024-11-07T12:26:01,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:26:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:01,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T12:26:01,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982421722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982421725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:01,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f5a74d16ce414dc193b851463315f25c_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:01,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742436_1612 (size=12304) 2024-11-07T12:26:01,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982421831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982421831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982422041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982422041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,164 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:26:02,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f5a74d16ce414dc193b851463315f25c_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f5a74d16ce414dc193b851463315f25c_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:02,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81bd4492af3c4281a2ea9c6b26d7771b, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:02,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81bd4492af3c4281a2ea9c6b26d7771b is 175, key is test_row_0/A:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742437_1613 (size=31105) 2024-11-07T12:26:02,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982422351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982422351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81bd4492af3c4281a2ea9c6b26d7771b 2024-11-07T12:26:02,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/ea8e9f914c724c419639c6f0e65b0f4e is 50, key is test_row_0/B:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:02,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742438_1614 (size=12151) 2024-11-07T12:26:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982422861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982422861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T12:26:02,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:26:02,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T12:26:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T12:26:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:02,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T12:26:02,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T12:26:02,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T12:26:02,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/ea8e9f914c724c419639c6f0e65b0f4e 2024-11-07T12:26:02,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ff1ee1a759a94714ae31ad8ea1ab20a1 is 50, key is test_row_0/C:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742439_1615 (size=12151) 2024-11-07T12:26:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:03,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:26:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:26:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:26:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:26:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T12:26:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1730982423351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:03,353 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_2' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:26:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:03,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54664 deadline: 1730982423371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:03,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54624 deadline: 1730982423371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:03,373 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_1' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:26:03,373 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8944cbb06cd2,46221,1730982182218 for 'test_row_0' in fb027f18a5ca907ae6c3eafe4c3b75bd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:03,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ff1ee1a759a94714ae31ad8ea1ab20a1 2024-11-07T12:26:03,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/81bd4492af3c4281a2ea9c6b26d7771b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b 2024-11-07T12:26:03,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b, entries=150, sequenceid=236, filesize=30.4 K 2024-11-07T12:26:03,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/ea8e9f914c724c419639c6f0e65b0f4e as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e 2024-11-07T12:26:03,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T12:26:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/ff1ee1a759a94714ae31ad8ea1ab20a1 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1 2024-11-07T12:26:03,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T12:26:03,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1695ms, sequenceid=236, compaction requested=true 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:03,398 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:03,398 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:26:03,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:26:03,399 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,399 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,399 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=91.6 K 2024-11-07T12:26:03,399 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6ee288e25c5e433cbc284fff536bf462, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=36.0 K 2024-11-07T12:26:03,399 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b] 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee288e25c5e433cbc284fff536bf462, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e0a80f0ad4243a8b451446fb74df3c6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:26:03,399 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1372277ed5407182d3f196e3943930, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730982358350 2024-11-07T12:26:03,400 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f34121aa3f7451db349e4ea4bf39ef5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730982358350 2024-11-07T12:26:03,400 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81bd4492af3c4281a2ea9c6b26d7771b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:03,400 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8e9f914c724c419639c6f0e65b0f4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:03,404 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:03,405 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:03,405 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/616909febd564651a89e1d744766325f is 50, key is test_row_0/B:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:03,406 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411077a07ffd5caa947b39861b9e81c335bd4_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:03,408 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411077a07ffd5caa947b39861b9e81c335bd4_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:03,408 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077a07ffd5caa947b39861b9e81c335bd4_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:03,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742441_1617 (size=4469) 2024-11-07T12:26:03,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742440_1616 (size=12697) 2024-11-07T12:26:03,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46221 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T12:26:03,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,430 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T12:26:03,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ecbb432292849ed9c5cbdbe9bec5de2_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982361721/Put/seqid=0 2024-11-07T12:26:03,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742442_1618 (size=12304) 2024-11-07T12:26:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:03,812 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:03,813 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/0bb0b7f8ce91417f9286f760a9c2e7ed is 175, key is test_row_0/A:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:03,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742443_1619 (size=31651) 2024-11-07T12:26:03,821 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/616909febd564651a89e1d744766325f as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/616909febd564651a89e1d744766325f 2024-11-07T12:26:03,825 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into 616909febd564651a89e1d744766325f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:03,825 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:03,825 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=13, startTime=1730982363398; duration=0sec 2024-11-07T12:26:03,825 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:26:03,825 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:26:03,825 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:03,826 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:03,826 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:26:03,826 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:03,826 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ec2605f9d3ab49f2b4b3f78028e8aa06, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=36.0 K 2024-11-07T12:26:03,826 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2605f9d3ab49f2b4b3f78028e8aa06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730982357185 2024-11-07T12:26:03,826 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d4736b5f447b45dc9eda8ae730230d13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730982358350 2024-11-07T12:26:03,827 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1ee1a759a94714ae31ad8ea1ab20a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:03,832 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:03,833 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/fbf999edd3604e609a5e24aa2586bbaa is 50, key is test_row_0/C:col10/1730982361702/Put/seqid=0 2024-11-07T12:26:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:26:03,841 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ecbb432292849ed9c5cbdbe9bec5de2_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ecbb432292849ed9c5cbdbe9bec5de2_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8fb3bb212f0e40b194d59b236c47dc3a, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:03,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8fb3bb212f0e40b194d59b236c47dc3a is 175, key is test_row_0/A:col10/1730982361721/Put/seqid=0 2024-11-07T12:26:03,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742444_1620 (size=12697) 2024-11-07T12:26:03,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742445_1621 (size=31105) 2024-11-07T12:26:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:03,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. as already flushing 2024-11-07T12:26:03,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982423906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982423907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,008 INFO [Thread-2468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2468. 2024-11-07T12:26:04,008 DEBUG [Thread-2468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:04,008 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,008 DEBUG [Thread-2468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:04,009 INFO [Thread-2470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2470. 2024-11-07T12:26:04,009 DEBUG [Thread-2470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:04,009 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,009 DEBUG [Thread-2470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:04,009 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-07T12:26:04,010 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:04,010 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,010 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:04,011 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-11-07T12:26:04,011 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:04,011 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,012 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:04,012 INFO [Thread-2472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2472. 2024-11-07T12:26:04,012 DEBUG [Thread-2472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:04,012 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,012 DEBUG [Thread-2472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:04,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982424021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982424021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:04,226 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/0bb0b7f8ce91417f9286f760a9c2e7ed as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed 2024-11-07T12:26:04,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982424231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982424231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,233 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into 0bb0b7f8ce91417f9286f760a9c2e7ed(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:04,233 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:04,233 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=13, startTime=1730982363398; duration=0sec 2024-11-07T12:26:04,234 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:04,234 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:26:04,247 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8fb3bb212f0e40b194d59b236c47dc3a 2024-11-07T12:26:04,253 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/fbf999edd3604e609a5e24aa2586bbaa as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/fbf999edd3604e609a5e24aa2586bbaa 2024-11-07T12:26:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/efaa1155291f40559099ecca43576fab is 50, key is test_row_0/B:col10/1730982361721/Put/seqid=0 2024-11-07T12:26:04,258 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into fbf999edd3604e609a5e24aa2586bbaa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:04,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:04,258 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=13, startTime=1730982363398; duration=0sec 2024-11-07T12:26:04,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:04,258 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:26:04,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742446_1622 (size=12151) 2024-11-07T12:26:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982424542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982424542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:04,662 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/efaa1155291f40559099ecca43576fab 2024-11-07T12:26:04,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d36a1c09843b44a19c24a7e799a5cae7 is 50, key is test_row_0/C:col10/1730982361721/Put/seqid=0 2024-11-07T12:26:04,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742447_1623 (size=12151) 2024-11-07T12:26:05,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:05,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T12:26:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54676 deadline: 1730982425051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46221 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1730982425051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 , the old value is region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., hostname=8944cbb06cd2,46221,1730982182218, seqNum=5 because the exception is null or not the one we care about 2024-11-07T12:26:05,078 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d36a1c09843b44a19c24a7e799a5cae7 2024-11-07T12:26:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/8fb3bb212f0e40b194d59b236c47dc3a as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a 2024-11-07T12:26:05,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a, entries=150, sequenceid=251, filesize=30.4 K 2024-11-07T12:26:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/efaa1155291f40559099ecca43576fab as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab 2024-11-07T12:26:05,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:26:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/d36a1c09843b44a19c24a7e799a5cae7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7 2024-11-07T12:26:05,102 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T12:26:05,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1673ms, sequenceid=251, compaction requested=false 2024-11-07T12:26:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8944cbb06cd2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T12:26:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T12:26:05,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T12:26:05,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1310 sec 2024-11-07T12:26:05,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1350 sec 2024-11-07T12:26:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T12:26:05,131 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:26:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46221 {}] regionserver.HRegion(8855): Flush requested on fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:06,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T12:26:06,068 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-11-07T12:26:06,068 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:06,068 INFO [Thread-2461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2461. 2024-11-07T12:26:06,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T12:26:06,068 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:06,068 DEBUG [Thread-2461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:06,068 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:06,068 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:06,068 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:06,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:06,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:06,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c431c720ca754ecdac4b00fb01025e1b_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:06,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742448_1624 (size=12454) 2024-11-07T12:26:06,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:26:06,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c431c720ca754ecdac4b00fb01025e1b_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c431c720ca754ecdac4b00fb01025e1b_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:06,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/028b612023b64154a0690259fc2f5084, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:06,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/028b612023b64154a0690259fc2f5084 is 175, key is test_row_0/A:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742449_1625 (size=31255) 2024-11-07T12:26:06,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/028b612023b64154a0690259fc2f5084 2024-11-07T12:26:06,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/97f3a62d22ba40f0a0b6b59accd0f4f4 is 50, key is test_row_0/B:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742450_1626 (size=12301) 2024-11-07T12:26:07,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/97f3a62d22ba40f0a0b6b59accd0f4f4 2024-11-07T12:26:07,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/98eeec081a624db8a6db67669612b1e6 is 50, key is test_row_0/C:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:07,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742451_1627 (size=12301) 2024-11-07T12:26:07,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/98eeec081a624db8a6db67669612b1e6 2024-11-07T12:26:07,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/028b612023b64154a0690259fc2f5084 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084 2024-11-07T12:26:07,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084, entries=150, sequenceid=277, filesize=30.5 K 2024-11-07T12:26:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/97f3a62d22ba40f0a0b6b59accd0f4f4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4 2024-11-07T12:26:07,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4, entries=150, sequenceid=277, filesize=12.0 K 2024-11-07T12:26:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/98eeec081a624db8a6db67669612b1e6 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6 2024-11-07T12:26:07,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6, entries=150, sequenceid=277, filesize=12.0 K 2024-11-07T12:26:07,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1679ms, sequenceid=277, compaction requested=true 2024-11-07T12:26:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T12:26:07,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:07,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:07,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:07,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb027f18a5ca907ae6c3eafe4c3b75bd:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:07,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/A is initiating minor compaction (all files) 2024-11-07T12:26:07,748 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/B is initiating minor compaction (all files) 2024-11-07T12:26:07,748 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/B in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:07,748 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/A in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:07,749 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/616909febd564651a89e1d744766325f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=36.3 K 2024-11-07T12:26:07,749 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=91.8 K 2024-11-07T12:26:07,749 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. files: [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084] 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 616909febd564651a89e1d744766325f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb0b7f8ce91417f9286f760a9c2e7ed, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting efaa1155291f40559099ecca43576fab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982361720 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fb3bb212f0e40b194d59b236c47dc3a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982361720 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f3a62d22ba40f0a0b6b59accd0f4f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730982363906 2024-11-07T12:26:07,749 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028b612023b64154a0690259fc2f5084, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730982363906 2024-11-07T12:26:07,755 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:07,755 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#B#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:07,756 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/41dcf1973352409dbedfaa09443968d7 is 50, key is test_row_0/B:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:07,756 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107950b2ee112b24c4b9359a97311625b7b_fb027f18a5ca907ae6c3eafe4c3b75bd store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:07,758 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107950b2ee112b24c4b9359a97311625b7b_fb027f18a5ca907ae6c3eafe4c3b75bd, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:07,758 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107950b2ee112b24c4b9359a97311625b7b_fb027f18a5ca907ae6c3eafe4c3b75bd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:07,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742452_1628 (size=12949) 2024-11-07T12:26:07,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742453_1629 (size=4469) 2024-11-07T12:26:08,163 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#A#compaction#523 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:08,164 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7e571a8f7ea448c6b1a4705ac9804953 is 175, key is test_row_0/A:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:08,169 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/41dcf1973352409dbedfaa09443968d7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/41dcf1973352409dbedfaa09443968d7 2024-11-07T12:26:08,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742454_1630 (size=31903) 2024-11-07T12:26:08,174 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/7e571a8f7ea448c6b1a4705ac9804953 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7e571a8f7ea448c6b1a4705ac9804953 2024-11-07T12:26:08,174 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/B of fb027f18a5ca907ae6c3eafe4c3b75bd into 41dcf1973352409dbedfaa09443968d7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:08,175 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/B, priority=13, startTime=1730982367747; duration=0sec 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:B 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T12:26:08,175 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1541): fb027f18a5ca907ae6c3eafe4c3b75bd/C is initiating minor compaction (all files) 2024-11-07T12:26:08,175 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb027f18a5ca907ae6c3eafe4c3b75bd/C in TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:08,175 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/fbf999edd3604e609a5e24aa2586bbaa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6] into tmpdir=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp, totalSize=36.3 K 2024-11-07T12:26:08,176 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf999edd3604e609a5e24aa2586bbaa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730982360553 2024-11-07T12:26:08,176 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting d36a1c09843b44a19c24a7e799a5cae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730982361720 2024-11-07T12:26:08,176 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] compactions.Compactor(225): Compacting 98eeec081a624db8a6db67669612b1e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1730982363906 2024-11-07T12:26:08,178 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/A of fb027f18a5ca907ae6c3eafe4c3b75bd into 7e571a8f7ea448c6b1a4705ac9804953(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:08,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:08,178 INFO [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/A, priority=13, startTime=1730982367747; duration=0sec 2024-11-07T12:26:08,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:08,178 DEBUG [RS:0;8944cbb06cd2:46221-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:A 2024-11-07T12:26:08,182 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb027f18a5ca907ae6c3eafe4c3b75bd#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T12:26:08,182 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/c8588e9311b84568b1d2dafbb85661bb is 50, key is test_row_0/C:col10/1730982366063/Put/seqid=0 2024-11-07T12:26:08,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742455_1631 (size=12949) 2024-11-07T12:26:08,596 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/c8588e9311b84568b1d2dafbb85661bb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/c8588e9311b84568b1d2dafbb85661bb 2024-11-07T12:26:08,602 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb027f18a5ca907ae6c3eafe4c3b75bd/C of fb027f18a5ca907ae6c3eafe4c3b75bd into c8588e9311b84568b1d2dafbb85661bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T12:26:08,602 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb027f18a5ca907ae6c3eafe4c3b75bd: 2024-11-07T12:26:08,602 INFO [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., storeName=fb027f18a5ca907ae6c3eafe4c3b75bd/C, priority=13, startTime=1730982367748; duration=0sec 2024-11-07T12:26:08,603 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T12:26:08,603 DEBUG [RS:0;8944cbb06cd2:46221-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb027f18a5ca907ae6c3eafe4c3b75bd:C 2024-11-07T12:26:13,383 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-11-07T12:26:13,383 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:13,383 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,384 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:13,443 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-11-07T12:26:13,443 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:13,443 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,443 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:13,472 INFO [Thread-2459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2459. 2024-11-07T12:26:13,472 DEBUG [Thread-2459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T12:26:13,472 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,472 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4370 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4229 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4289 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4378 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4221 2024-11-07T12:26:13,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T12:26:13,473 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:26:13,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:26:13,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:13,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:13,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T12:26:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:13,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982373477"}]},"ts":"1730982373477"} 2024-11-07T12:26:13,479 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T12:26:13,479 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T12:26:13,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T12:26:13,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, UNASSIGN}] 2024-11-07T12:26:13,483 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, UNASSIGN 2024-11-07T12:26:13,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=CLOSING, regionLocation=8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:13,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, UNASSIGN because future has completed 2024-11-07T12:26:13,486 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T12:26:13,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218}] 2024-11-07T12:26:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:13,640 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:13,640 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T12:26:13,640 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing fb027f18a5ca907ae6c3eafe4c3b75bd, disabling compactions & flushes 2024-11-07T12:26:13,640 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:13,640 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:13,640 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. after waiting 0 ms 2024-11-07T12:26:13,640 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:13,641 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing fb027f18a5ca907ae6c3eafe4c3b75bd 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T12:26:13,651 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070ee1a3320a60427f8f08734afb519c5b_fb027f18a5ca907ae6c3eafe4c3b75bd is 50, key is test_row_0/A:col10/1730982373471/Put/seqid=0 2024-11-07T12:26:13,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742456_1632 (size=12454) 2024-11-07T12:26:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:14,055 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T12:26:14,058 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070ee1a3320a60427f8f08734afb519c5b_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ee1a3320a60427f8f08734afb519c5b_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:14,059 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/cd1caaa725e04a619dc778bdbd75996b, store: [table=TestAcidGuarantees family=A region=fb027f18a5ca907ae6c3eafe4c3b75bd] 2024-11-07T12:26:14,059 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/cd1caaa725e04a619dc778bdbd75996b is 175, key is test_row_0/A:col10/1730982373471/Put/seqid=0 2024-11-07T12:26:14,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742457_1633 (size=31255) 2024-11-07T12:26:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:14,465 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/cd1caaa725e04a619dc778bdbd75996b 2024-11-07T12:26:14,477 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/697ba149ef4e490f8b6fc2d8dd1b1765 is 50, key is test_row_0/B:col10/1730982373471/Put/seqid=0 2024-11-07T12:26:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742458_1634 (size=12301) 2024-11-07T12:26:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:14,881 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/697ba149ef4e490f8b6fc2d8dd1b1765 2024-11-07T12:26:14,892 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/8fa89b84e9d048f9be85bdd8ad5b7c43 is 50, key is test_row_0/C:col10/1730982373471/Put/seqid=0 2024-11-07T12:26:14,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742459_1635 (size=12301) 2024-11-07T12:26:15,297 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/8fa89b84e9d048f9be85bdd8ad5b7c43 2024-11-07T12:26:15,306 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/A/cd1caaa725e04a619dc778bdbd75996b as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/cd1caaa725e04a619dc778bdbd75996b 2024-11-07T12:26:15,311 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/cd1caaa725e04a619dc778bdbd75996b, entries=150, sequenceid=286, filesize=30.5 K 2024-11-07T12:26:15,312 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/B/697ba149ef4e490f8b6fc2d8dd1b1765 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/697ba149ef4e490f8b6fc2d8dd1b1765 2024-11-07T12:26:15,317 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/697ba149ef4e490f8b6fc2d8dd1b1765, entries=150, sequenceid=286, filesize=12.0 K 2024-11-07T12:26:15,318 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/.tmp/C/8fa89b84e9d048f9be85bdd8ad5b7c43 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8fa89b84e9d048f9be85bdd8ad5b7c43 2024-11-07T12:26:15,321 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8fa89b84e9d048f9be85bdd8ad5b7c43, entries=150, sequenceid=286, filesize=12.0 K 2024-11-07T12:26:15,322 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1682ms, sequenceid=286, compaction requested=false 2024-11-07T12:26:15,322 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084] to archive 2024-11-07T12:26:15,323 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:26:15,324 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/191626b859724f5eb1eb074e26e4ebf1 2024-11-07T12:26:15,325 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/ed34c46c7daf4700bf616af69ac193f8 2024-11-07T12:26:15,326 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fde27bb3124b4b42890e5fbc19cb0bdd 2024-11-07T12:26:15,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7817fffef8874ea38595035c6d041ecb 2024-11-07T12:26:15,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/46e0bc5dbd494a95adc461026a8fdfda 2024-11-07T12:26:15,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/86772dd5b05d42f19401e8b668900e40 2024-11-07T12:26:15,329 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/fb52634bb22e4f1d9194e9a6b062f5cc 2024-11-07T12:26:15,330 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2546b5461d8b4a909aa3d86410c863cd 2024-11-07T12:26:15,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/2fd3098aacf347b9a2ba2dd8cbf8dbfc 2024-11-07T12:26:15,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/d9e34322447642d8a7783982922fa890 2024-11-07T12:26:15,332 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81d1f36b6b6e4eea8c4d9cc033639783 2024-11-07T12:26:15,333 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/b4c3a0670ab149d6995b78065285fcff 2024-11-07T12:26:15,334 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/a488f0778169490bbd71e55124667060 2024-11-07T12:26:15,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8e0a80f0ad4243a8b451446fb74df3c6 2024-11-07T12:26:15,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/dd1372277ed5407182d3f196e3943930 2024-11-07T12:26:15,336 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/0bb0b7f8ce91417f9286f760a9c2e7ed 2024-11-07T12:26:15,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/81bd4492af3c4281a2ea9c6b26d7771b 2024-11-07T12:26:15,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/8fb3bb212f0e40b194d59b236c47dc3a 2024-11-07T12:26:15,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/028b612023b64154a0690259fc2f5084 2024-11-07T12:26:15,339 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fc9301bab4944148af965dce0bb80c73, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/512e9ad4096c467491ec048890e9c765, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/78cbd25a9f264afe8f78374b346b608a, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6ee288e25c5e433cbc284fff536bf462, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/616909febd564651a89e1d744766325f, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4] to archive 2024-11-07T12:26:15,340 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:26:15,340 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1058be1aa18448bc86e81bd7cd072b3d 2024-11-07T12:26:15,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fd1a1da12eaf4cbfac758205ad9e3228 2024-11-07T12:26:15,342 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fc9301bab4944148af965dce0bb80c73 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/fc9301bab4944148af965dce0bb80c73 2024-11-07T12:26:15,342 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/a8d07257e3924dafa3f6ff4bd1ad403a 2024-11-07T12:26:15,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/21edf8f4074e4440bb7a5788fa101af7 2024-11-07T12:26:15,344 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/512e9ad4096c467491ec048890e9c765 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/512e9ad4096c467491ec048890e9c765 2024-11-07T12:26:15,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/4beddaafc3df41c89bc20acfbb69eb59 2024-11-07T12:26:15,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e1e8e58f9bef4f28bb499501ba91520c 2024-11-07T12:26:15,346 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/78cbd25a9f264afe8f78374b346b608a to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/78cbd25a9f264afe8f78374b346b608a 2024-11-07T12:26:15,347 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/1320d2ac4c744b2db8f7e9c2f1424908 2024-11-07T12:26:15,348 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/04cbc2bf920d48c2ae8d4cc739c953e4 2024-11-07T12:26:15,349 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/e600814dd4ce4df5824045ea74b35b6e 2024-11-07T12:26:15,350 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6ee288e25c5e433cbc284fff536bf462 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6ee288e25c5e433cbc284fff536bf462 2024-11-07T12:26:15,351 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/cb16919fd02d4ba287a413c2aacbf5be 2024-11-07T12:26:15,351 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/6f34121aa3f7451db349e4ea4bf39ef5 2024-11-07T12:26:15,352 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/616909febd564651a89e1d744766325f to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/616909febd564651a89e1d744766325f 2024-11-07T12:26:15,353 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/ea8e9f914c724c419639c6f0e65b0f4e 2024-11-07T12:26:15,354 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/efaa1155291f40559099ecca43576fab 2024-11-07T12:26:15,355 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/97f3a62d22ba40f0a0b6b59accd0f4f4 2024-11-07T12:26:15,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ffa0a08819e142d2a0a1fe041d6574a5, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/23c58f3464274ade8b6f1d4d0b52be07, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8f606cfc04524491a6a46a93c4bc04cc, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ec2605f9d3ab49f2b4b3f78028e8aa06, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/fbf999edd3604e609a5e24aa2586bbaa, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6] to archive 2024-11-07T12:26:15,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T12:26:15,357 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d06c65a93cb1437a86bc5172fd9657b7 2024-11-07T12:26:15,358 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d939df2ddfd14ce5880bdea9a86a55d0 2024-11-07T12:26:15,359 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ffa0a08819e142d2a0a1fe041d6574a5 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ffa0a08819e142d2a0a1fe041d6574a5 2024-11-07T12:26:15,359 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/73d84e2e51a74e8d88e08f5fff0e51e8 2024-11-07T12:26:15,360 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e3a03316122d4aa8824f08d8d080d068 2024-11-07T12:26:15,361 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/23c58f3464274ade8b6f1d4d0b52be07 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/23c58f3464274ade8b6f1d4d0b52be07 2024-11-07T12:26:15,361 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/2f91068d812c4490b4f62c270e62f874 2024-11-07T12:26:15,362 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/3dd7acbd291e41c28e87460eb4a49487 2024-11-07T12:26:15,363 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8f606cfc04524491a6a46a93c4bc04cc to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8f606cfc04524491a6a46a93c4bc04cc 2024-11-07T12:26:15,363 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/24d18255d8724d3eb1562514c681aa2d 2024-11-07T12:26:15,364 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ae3640f9b50c4cabb69a4771657ac921 2024-11-07T12:26:15,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/be29a315085a4875a31335bb105424d7 2024-11-07T12:26:15,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ec2605f9d3ab49f2b4b3f78028e8aa06 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ec2605f9d3ab49f2b4b3f78028e8aa06 2024-11-07T12:26:15,366 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/e41401e5e60648ea8289414c826d0d8d 2024-11-07T12:26:15,367 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d4736b5f447b45dc9eda8ae730230d13 2024-11-07T12:26:15,368 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/fbf999edd3604e609a5e24aa2586bbaa to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/fbf999edd3604e609a5e24aa2586bbaa 2024-11-07T12:26:15,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/ff1ee1a759a94714ae31ad8ea1ab20a1 2024-11-07T12:26:15,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/d36a1c09843b44a19c24a7e799a5cae7 2024-11-07T12:26:15,370 DEBUG [StoreCloser-TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/98eeec081a624db8a6db67669612b1e6 2024-11-07T12:26:15,374 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits/289.seqid, newMaxSeqId=289, maxSeqId=4 2024-11-07T12:26:15,375 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. 2024-11-07T12:26:15,375 DEBUG [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for fb027f18a5ca907ae6c3eafe4c3b75bd: Waiting for close lock at 1730982373640Running coprocessor pre-close hooks at 1730982373640Disabling compacts and flushes for region at 1730982373640Disabling writes for close at 1730982373640Obtaining lock to block concurrent updates at 1730982373641 (+1 ms)Preparing flush snapshotting stores in fb027f18a5ca907ae6c3eafe4c3b75bd at 1730982373641Finished memstore snapshotting TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730982373641Flushing stores of TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd. at 1730982373643 (+2 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/A: creating writer at 1730982373643Flushing mob file fb027f18a5ca907ae6c3eafe4c3b75bd/A: appending metadata at 1730982373651 (+8 ms)Flushing mob file fb027f18a5ca907ae6c3eafe4c3b75bd/A: closing flushed file at 1730982373651Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/A: appending metadata at 1730982374059 (+408 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/A: closing flushed file at 1730982374059Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/B: creating writer at 1730982374474 (+415 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/B: appending metadata at 1730982374477 (+3 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/B: closing flushed file at 1730982374477Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/C: creating writer at 1730982374888 (+411 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/C: appending metadata at 1730982374891 (+3 ms)Flushing fb027f18a5ca907ae6c3eafe4c3b75bd/C: closing flushed file at 1730982374891Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6247a9c7: reopening flushed file at 1730982375305 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c2221aa: reopening flushed file at 1730982375311 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@100f99e8: reopening flushed file at 1730982375317 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for fb027f18a5ca907ae6c3eafe4c3b75bd in 1682ms, sequenceid=286, compaction requested=false at 1730982375322 (+5 ms)Writing region close event to WAL at 1730982375371 (+49 ms)Running coprocessor post-close hooks at 1730982375375 (+4 ms)Closed at 1730982375375 2024-11-07T12:26:15,377 INFO [RS_CLOSE_REGION-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fb027f18a5ca907ae6c3eafe4c3b75bd, regionState=CLOSED 2024-11-07T12:26:15,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 because future has completed 2024-11-07T12:26:15,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T12:26:15,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure fb027f18a5ca907ae6c3eafe4c3b75bd, server=8944cbb06cd2,46221,1730982182218 in 1.8940 sec 2024-11-07T12:26:15,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-07T12:26:15,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb027f18a5ca907ae6c3eafe4c3b75bd, UNASSIGN in 1.9010 sec 2024-11-07T12:26:15,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T12:26:15,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-07T12:26:15,388 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730982375388"}]},"ts":"1730982375388"} 2024-11-07T12:26:15,389 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T12:26:15,389 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T12:26:15,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-07T12:26:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T12:26:15,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:26:15,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T12:26:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,638 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:26:15,640 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,645 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,648 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C, FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits] 2024-11-07T12:26:15,651 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7e571a8f7ea448c6b1a4705ac9804953 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/7e571a8f7ea448c6b1a4705ac9804953 2024-11-07T12:26:15,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/cd1caaa725e04a619dc778bdbd75996b to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/A/cd1caaa725e04a619dc778bdbd75996b 2024-11-07T12:26:15,656 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/41dcf1973352409dbedfaa09443968d7 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/41dcf1973352409dbedfaa09443968d7 2024-11-07T12:26:15,658 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/697ba149ef4e490f8b6fc2d8dd1b1765 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/B/697ba149ef4e490f8b6fc2d8dd1b1765 2024-11-07T12:26:15,661 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8fa89b84e9d048f9be85bdd8ad5b7c43 to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/8fa89b84e9d048f9be85bdd8ad5b7c43 2024-11-07T12:26:15,663 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/c8588e9311b84568b1d2dafbb85661bb to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/C/c8588e9311b84568b1d2dafbb85661bb 2024-11-07T12:26:15,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits/289.seqid to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd/recovered.edits/289.seqid 2024-11-07T12:26:15,667 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/default/TestAcidGuarantees/fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,667 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T12:26:15,668 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:26:15,669 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T12:26:15,673 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ee1a3320a60427f8f08734afb519c5b_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ee1a3320a60427f8f08734afb519c5b_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,675 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071ad2f7c544df453c9926a9a17b8d40e3_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071ad2f7c544df453c9926a9a17b8d40e3_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,677 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072906d65119eb479dbf94c8c656ddf3de_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072906d65119eb479dbf94c8c656ddf3de_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,680 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072f4f5a7df4964fe28dc574312e679875_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072f4f5a7df4964fe28dc574312e679875_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,682 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107346ec64b9bd741789fa5efe448224b7a_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107346ec64b9bd741789fa5efe448224b7a_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,684 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743b4eb941efe4694b5992bd1fc73250f_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743b4eb941efe4694b5992bd1fc73250f_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,685 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a336a6c18f94ad69c90bfa73f6566ce_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a336a6c18f94ad69c90bfa73f6566ce_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,687 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ecbb432292849ed9c5cbdbe9bec5de2_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ecbb432292849ed9c5cbdbe9bec5de2_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,689 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d516c655d4d41fb8d4aaef341ac35c2_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078d516c655d4d41fb8d4aaef341ac35c2_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,690 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110793d7168433a84db48f56e3c43c9a6f37_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110793d7168433a84db48f56e3c43c9a6f37_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,692 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a120578d63d4a95a807019413fff81a_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079a120578d63d4a95a807019413fff81a_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,693 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0bae90bdf43427c81affa1da6119e45_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0bae90bdf43427c81affa1da6119e45_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,695 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c431c720ca754ecdac4b00fb01025e1b_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c431c720ca754ecdac4b00fb01025e1b_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,696 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca22c4be549f4bbb8d73b6a645c8acef_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca22c4be549f4bbb8d73b6a645c8acef_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,697 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f5a74d16ce414dc193b851463315f25c_fb027f18a5ca907ae6c3eafe4c3b75bd to hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f5a74d16ce414dc193b851463315f25c_fb027f18a5ca907ae6c3eafe4c3b75bd 2024-11-07T12:26:15,698 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T12:26:15,699 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,701 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T12:26:15,702 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T12:26:15,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T12:26:15,703 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730982375703"}]},"ts":"9223372036854775807"} 2024-11-07T12:26:15,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T12:26:15,705 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fb027f18a5ca907ae6c3eafe4c3b75bd, NAME => 'TestAcidGuarantees,,1730982340874.fb027f18a5ca907ae6c3eafe4c3b75bd.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T12:26:15,705 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T12:26:15,705 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730982375705"}]},"ts":"9223372036854775807"} 2024-11-07T12:26:15,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T12:26:15,707 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T12:26:15,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-07T12:26:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T12:26:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T12:26:15,752 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T12:26:15,766 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 238), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=379 (was 482), ProcessCount=11 (was 11), AvailableMemoryMB=1477 (was 1513) 2024-11-07T12:26:15,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T12:26:15,766 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:26:15,766 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:26:15,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:15,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:15,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T12:26:15,767 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T12:26:15,767 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-07T12:26:15,768 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8944cbb06cd2,37285,1730982181418 2024-11-07T12:26:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:26:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T12:26:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:26:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:26:15,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:26:15,838 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T12:26:15,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:26:15,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:26:15,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:15,839 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T12:26:15,840 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8944cbb06cd2,46221,1730982182218' ***** 2024-11-07T12:26:15,840 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T12:26:15,841 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T12:26:15,842 INFO [RS:0;8944cbb06cd2:46221 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T12:26:15,842 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T12:26:15,842 INFO [RS:0;8944cbb06cd2:46221 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T12:26:15,842 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(959): stopping server 8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:15,842 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:26:15,843 INFO [RS:0;8944cbb06cd2:46221 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8944cbb06cd2:46221. 2024-11-07T12:26:15,843 DEBUG [RS:0;8944cbb06cd2:46221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T12:26:15,843 DEBUG [RS:0;8944cbb06cd2:46221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:15,843 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T12:26:15,843 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T12:26:15,843 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T12:26:15,843 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T12:26:15,844 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T12:26:15,844 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T12:26:15,844 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:15,844 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T12:26:15,844 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T12:26:15,844 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T12:26:15,844 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T12:26:15,844 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T12:26:15,844 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T12:26:15,856 INFO [regionserver/8944cbb06cd2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:26:15,864 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/info/195820b3d7c749a681e1fd4fa4a00ead is 95, key is TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef./info:/1730982219334/DeleteFamily/seqid=0 2024-11-07T12:26:15,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742460_1636 (size=6514) 2024-11-07T12:26:16,044 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:16,222 INFO [regionserver/8944cbb06cd2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T12:26:16,223 INFO [regionserver/8944cbb06cd2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T12:26:16,244 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:16,268 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/info/195820b3d7c749a681e1fd4fa4a00ead 2024-11-07T12:26:16,291 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/ns/2d787dc675524a819c154b9524d9acdc is 93, key is TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef./ns:/1730982219300/DeleteFamily/seqid=0 2024-11-07T12:26:16,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742461_1637 (size=5906) 2024-11-07T12:26:16,445 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:16,645 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:16,696 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/ns/2d787dc675524a819c154b9524d9acdc 2024-11-07T12:26:16,725 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/rep_barrier/79c6b24def8641d0bbf64c52cd780d55 is 102, key is TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef./rep_barrier:/1730982219300/DeleteFamily/seqid=0 2024-11-07T12:26:16,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742462_1638 (size=6025) 2024-11-07T12:26:16,845 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:17,046 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:17,130 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/rep_barrier/79c6b24def8641d0bbf64c52cd780d55 2024-11-07T12:26:17,200 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/table/7f546f4f7e37437a996378c877eea10d is 96, key is TestAcidGuarantees,,1730982186930.533db45226dd865e0c0b2154b214ffef./table:/1730982219300/DeleteFamily/seqid=0 2024-11-07T12:26:17,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742463_1639 (size=5936) 2024-11-07T12:26:17,246 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:17,446 DEBUG [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T12:26:17,605 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/table/7f546f4f7e37437a996378c877eea10d 2024-11-07T12:26:17,613 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/info/195820b3d7c749a681e1fd4fa4a00ead as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/info/195820b3d7c749a681e1fd4fa4a00ead 2024-11-07T12:26:17,619 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/info/195820b3d7c749a681e1fd4fa4a00ead, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T12:26:17,620 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/ns/2d787dc675524a819c154b9524d9acdc as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/ns/2d787dc675524a819c154b9524d9acdc 2024-11-07T12:26:17,625 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/ns/2d787dc675524a819c154b9524d9acdc, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T12:26:17,625 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/rep_barrier/79c6b24def8641d0bbf64c52cd780d55 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/rep_barrier/79c6b24def8641d0bbf64c52cd780d55 2024-11-07T12:26:17,629 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/rep_barrier/79c6b24def8641d0bbf64c52cd780d55, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T12:26:17,630 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/.tmp/table/7f546f4f7e37437a996378c877eea10d as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/table/7f546f4f7e37437a996378c877eea10d 2024-11-07T12:26:17,634 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/table/7f546f4f7e37437a996378c877eea10d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T12:26:17,635 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1791ms, sequenceid=90, compaction requested=false 2024-11-07T12:26:17,639 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T12:26:17,640 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T12:26:17,640 INFO [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T12:26:17,640 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730982375844Running coprocessor pre-close hooks at 1730982375844Disabling compacts and flushes for region at 1730982375844Disabling writes for close at 1730982375844Obtaining lock to block concurrent updates at 1730982375844Preparing flush snapshotting stores in 1588230740 at 1730982375844Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730982375845 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730982375845Flushing 1588230740/info: creating writer at 1730982375845Flushing 1588230740/info: appending metadata at 1730982375863 (+18 ms)Flushing 1588230740/info: closing flushed file at 1730982375863Flushing 1588230740/ns: creating writer at 1730982376277 (+414 ms)Flushing 1588230740/ns: appending metadata at 1730982376291 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1730982376291Flushing 1588230740/rep_barrier: creating writer at 1730982376706 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730982376725 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730982376725Flushing 1588230740/table: creating writer at 1730982377181 (+456 ms)Flushing 1588230740/table: appending metadata at 1730982377200 (+19 ms)Flushing 1588230740/table: closing flushed file at 1730982377200Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45a79dba: reopening flushed file at 1730982377612 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d2929a9: reopening flushed file at 1730982377619 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78761d9f: reopening flushed file at 1730982377625 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c13a8f7: reopening flushed file at 1730982377629 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1791ms, sequenceid=90, compaction requested=false at 1730982377635 (+6 ms)Writing region close event to WAL at 1730982377637 (+2 ms)Running coprocessor post-close hooks at 1730982377640 (+3 ms)Closed at 1730982377640 2024-11-07T12:26:17,640 DEBUG [RS_CLOSE_META-regionserver/8944cbb06cd2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T12:26:17,647 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(976): stopping server 8944cbb06cd2,46221,1730982182218; all regions closed. 2024-11-07T12:26:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741834_1010 (size=25344) 2024-11-07T12:26:17,656 DEBUG [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/oldWALs 2024-11-07T12:26:17,657 INFO [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8944cbb06cd2%2C46221%2C1730982182218.meta:.meta(num 1730982186497) 2024-11-07T12:26:17,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741832_1008 (size=14171867) 2024-11-07T12:26:17,663 DEBUG [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/oldWALs 2024-11-07T12:26:17,663 INFO [RS:0;8944cbb06cd2:46221 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8944cbb06cd2%2C46221%2C1730982182218:(num 1730982185357) 2024-11-07T12:26:17,663 DEBUG [RS:0;8944cbb06cd2:46221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T12:26:17,663 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T12:26:17,663 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:26:17,663 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.ChoreService(370): Chore service for: regionserver/8944cbb06cd2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T12:26:17,663 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:26:17,664 INFO [regionserver/8944cbb06cd2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:26:17,664 INFO [RS:0;8944cbb06cd2:46221 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46221 2024-11-07T12:26:17,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8944cbb06cd2,46221,1730982182218 2024-11-07T12:26:17,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T12:26:17,677 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:26:17,688 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8944cbb06cd2,46221,1730982182218] 2024-11-07T12:26:17,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8944cbb06cd2,46221,1730982182218 already deleted, retry=false 2024-11-07T12:26:17,698 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8944cbb06cd2,46221,1730982182218 expired; onlineServers=0 2024-11-07T12:26:17,698 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8944cbb06cd2,37285,1730982181418' ***** 2024-11-07T12:26:17,698 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T12:26:17,698 INFO [M:0;8944cbb06cd2:37285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T12:26:17,698 INFO [M:0;8944cbb06cd2:37285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T12:26:17,699 DEBUG [M:0;8944cbb06cd2:37285 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T12:26:17,699 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T12:26:17,699 DEBUG [M:0;8944cbb06cd2:37285 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T12:26:17,699 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.small.0-1730982185047 {}] cleaner.HFileCleaner(306): Exit Thread[master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.small.0-1730982185047,5,FailOnTimeoutGroup] 2024-11-07T12:26:17,699 DEBUG [master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.large.0-1730982185046 {}] cleaner.HFileCleaner(306): Exit Thread[master/8944cbb06cd2:0:becomeActiveMaster-HFileCleaner.large.0-1730982185046,5,FailOnTimeoutGroup] 2024-11-07T12:26:17,699 INFO [M:0;8944cbb06cd2:37285 {}] hbase.ChoreService(370): Chore service for: master/8944cbb06cd2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T12:26:17,699 INFO [M:0;8944cbb06cd2:37285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T12:26:17,699 DEBUG [M:0;8944cbb06cd2:37285 {}] master.HMaster(1795): Stopping service threads 2024-11-07T12:26:17,699 INFO [M:0;8944cbb06cd2:37285 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T12:26:17,700 INFO [M:0;8944cbb06cd2:37285 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T12:26:17,700 ERROR [M:0;8944cbb06cd2:37285 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:41983 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41983,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T12:26:17,701 INFO [M:0;8944cbb06cd2:37285 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T12:26:17,701 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T12:26:17,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T12:26:17,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T12:26:17,709 DEBUG [M:0;8944cbb06cd2:37285 {}] zookeeper.ZKUtil(347): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T12:26:17,709 WARN [M:0;8944cbb06cd2:37285 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T12:26:17,710 INFO [M:0;8944cbb06cd2:37285 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/.lastflushedseqids 2024-11-07T12:26:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742464_1640 (size=72) 2024-11-07T12:26:17,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:26:17,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46221-0x101153ff26c0001, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:26:17,789 INFO [RS:0;8944cbb06cd2:46221 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:26:17,789 INFO [RS:0;8944cbb06cd2:46221 {}] regionserver.HRegionServer(1031): Exiting; stopping=8944cbb06cd2,46221,1730982182218; zookeeper connection closed. 2024-11-07T12:26:17,790 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28cd41f4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28cd41f4 2024-11-07T12:26:17,791 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T12:26:18,125 INFO [M:0;8944cbb06cd2:37285 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T12:26:18,126 INFO [M:0;8944cbb06cd2:37285 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T12:26:18,126 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T12:26:18,126 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:26:18,126 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:26:18,126 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T12:26:18,126 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:26:18,126 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=731.43 KB heapSize=897.40 KB 2024-11-07T12:26:18,145 DEBUG [M:0;8944cbb06cd2:37285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ef32faae354ec3ad0138461a2151fb is 82, key is hbase:meta,,1/info:regioninfo/1730982186583/Put/seqid=0 2024-11-07T12:26:18,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742465_1641 (size=5672) 2024-11-07T12:26:18,549 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ef32faae354ec3ad0138461a2151fb 2024-11-07T12:26:18,578 DEBUG [M:0;8944cbb06cd2:37285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6652a3372f504327b74228cec3a066e7 is 2275, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1730982222506/Put/seqid=0 2024-11-07T12:26:18,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742466_1642 (size=43761) 2024-11-07T12:26:18,982 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=730.88 KB at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6652a3372f504327b74228cec3a066e7 2024-11-07T12:26:18,991 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6652a3372f504327b74228cec3a066e7 2024-11-07T12:26:19,009 DEBUG [M:0;8944cbb06cd2:37285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4bd72edc00564f7a9bb63977ac8d5dc4 is 69, key is 8944cbb06cd2,46221,1730982182218/rs:state/1730982185114/Put/seqid=0 2024-11-07T12:26:19,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073742467_1643 (size=5156) 2024-11-07T12:26:19,414 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4bd72edc00564f7a9bb63977ac8d5dc4 2024-11-07T12:26:19,424 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93ef32faae354ec3ad0138461a2151fb as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93ef32faae354ec3ad0138461a2151fb 2024-11-07T12:26:19,427 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93ef32faae354ec3ad0138461a2151fb, entries=8, sequenceid=2042, filesize=5.5 K 2024-11-07T12:26:19,427 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6652a3372f504327b74228cec3a066e7 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6652a3372f504327b74228cec3a066e7 2024-11-07T12:26:19,430 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6652a3372f504327b74228cec3a066e7 2024-11-07T12:26:19,430 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6652a3372f504327b74228cec3a066e7, entries=168, sequenceid=2042, filesize=42.7 K 2024-11-07T12:26:19,431 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4bd72edc00564f7a9bb63977ac8d5dc4 as hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4bd72edc00564f7a9bb63977ac8d5dc4 2024-11-07T12:26:19,434 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/af9a40c2-98ac-fd02-8bb6-fa22fd2805e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4bd72edc00564f7a9bb63977ac8d5dc4, entries=1, sequenceid=2042, filesize=5.0 K 2024-11-07T12:26:19,435 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(3140): Finished flush of dataSize ~731.43 KB/748988, heapSize ~897.10 KB/918632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2042, compaction requested=false 2024-11-07T12:26:19,436 INFO [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T12:26:19,436 DEBUG [M:0;8944cbb06cd2:37285 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730982378126Disabling compacts and flushes for region at 1730982378126Disabling writes for close at 1730982378126Obtaining lock to block concurrent updates at 1730982378126Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730982378126Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=748988, getHeapSize=918872, getOffHeapSize=0, getCellsCount=2235 at 1730982378127 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730982378129 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730982378129Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730982378145 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730982378145Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730982378559 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730982378577 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730982378577Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730982378991 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730982379009 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730982379009Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ffdb7ae: reopening flushed file at 1730982379423 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e185bc5: reopening flushed file at 1730982379427 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44d9b309: reopening flushed file at 1730982379431 (+4 ms)Finished flush of dataSize ~731.43 KB/748988, heapSize ~897.10 KB/918632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2042, compaction requested=false at 1730982379435 (+4 ms)Writing region close event to WAL at 1730982379436 (+1 ms)Closed at 1730982379436 2024-11-07T12:26:19,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43943 is added to blk_1073741830_1006 (size=882909) 2024-11-07T12:26:19,439 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T12:26:19,440 INFO [M:0;8944cbb06cd2:37285 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T12:26:19,440 INFO [M:0;8944cbb06cd2:37285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37285 2024-11-07T12:26:19,440 INFO [M:0;8944cbb06cd2:37285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T12:26:19,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:26:19,599 INFO [M:0;8944cbb06cd2:37285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T12:26:19,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37285-0x101153ff26c0000, quorum=127.0.0.1:51223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T12:26:19,608 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T12:26:19,612 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:26:19,612 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:26:19,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:26:19,613 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.log.dir/,STOPPED} 2024-11-07T12:26:19,615 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T12:26:19,615 WARN [BP-1409914116-172.17.0.2-1730982177397 heartbeating to localhost/127.0.0.1:41983 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T12:26:19,615 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T12:26:19,615 WARN [BP-1409914116-172.17.0.2-1730982177397 heartbeating to localhost/127.0.0.1:41983 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1409914116-172.17.0.2-1730982177397 (Datanode Uuid e5e0a903-46f3-4733-8cee-0a5f90087d4c) service to localhost/127.0.0.1:41983 2024-11-07T12:26:19,618 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data1/current/BP-1409914116-172.17.0.2-1730982177397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:26:19,618 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/cluster_457429c2-45a3-8340-00ad-074f1a0b0859/data/data2/current/BP-1409914116-172.17.0.2-1730982177397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T12:26:19,619 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T12:26:19,626 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T12:26:19,626 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T12:26:19,626 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T12:26:19,626 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T12:26:19,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9f0fc0c3-e648-87a3-691b-bd64bf33e6a5/hadoop.log.dir/,STOPPED} 2024-11-07T12:26:19,640 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T12:26:19,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down